From 85fca682d50571b9bc735d6d2de1547bbcbc1824 Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Thu, 8 Oct 2026 04:56:28 -0400 Subject: [PATCH] feat(s071): the Telegram gateway, and two fixes to the gateway core Slice 071. Every automatic criterion runs against a local fake Bot API. The live run needs the owner's bot token and is queued with exact steps. Two security defects in slice 070's core, each reproduced red first: - F1. Router.prompt_for/1 sent an unknown sender the very pairing code it was meant to read on /gateways, so anyone who found the bot could pair by echoing the prompt back. 070's own test asserted the leak. The prompt now says where the code is, never what it is, restoring docs/07's rule ("shown on /gateways and nowhere else"). - F2. Approval ids are UUIDv7, so the 8-character short id is shared by every request raised within about 65 seconds, and /approve decided whichever came first. Commands.find_pending/1 now refuses an ambiguous prefix, and buttons carry the full id. The gateway: - Bot API calls over Req, already in the lock: telegex has only an old release candidate and ex_gram keeps its offset in memory, so neither meets AC5. The token is read on every call and never stored; errors never carry the URL, which holds it; Req's retry step is off. - MarkdownV2 conversion that only emits a closed entity, with chunking checked after conversion and properties against an independent parse checker. - An outbox that throttles edits to at least 1 s and keeps the final text; long polling with the offset persisted before routing (at most once); photos fetched only after admission, size-capped and magic-checked, stored by digest; approval buttons only under the channel cap. - Trinity.Gateways.Supervisor starts the configured gateways, chosen by TRINITY_GATEWAYS; nothing in the app started a gateway before. - Images reach a session row and a :vision model, and are replaced by a note otherwise. The log filter and secret scanner learn the token's shape. Red evidence R1 to R10 is in the slice NOTES. Signed-off-by: Ayla Croft --- .env.example | 7 + CHANGELOG.md | 24 + README.md | 12 +- VERSIONS.md | 5 +- config/config.exs | 6 + config/llm.exs | 14 +- config/runtime.exs | 15 + docs/01-architecture.md | 10 +- docs/02-tech-stack.md | 2 +- docs/05-data-model.md | 1 + docs/07-security-model.md | 46 ++ lib/mix/tasks/trinity.secrets.scan.ex | 2 + lib/trinity/application.ex | 6 +- lib/trinity/gateways.ex | 26 +- lib/trinity/gateways/adapter.ex | 21 + lib/trinity/gateways/commands.ex | 13 +- lib/trinity/gateways/router.ex | 84 ++- lib/trinity/gateways/supervisor.ex | 32 ++ lib/trinity/gateways/telegram.ex | 133 +++++ lib/trinity/gateways/telegram/client.ex | 113 ++++ lib/trinity/gateways/telegram/markdown.ex | 308 +++++++++++ lib/trinity/gateways/telegram/media.ex | 127 +++++ lib/trinity/gateways/telegram/offset.ex | 51 ++ lib/trinity/gateways/telegram/outbox.ex | 257 ++++++++++ lib/trinity/gateways/telegram/poller.ex | 130 +++++ lib/trinity/gateways/telegram/updates.ex | 155 ++++++ lib/trinity/llm.ex | 11 +- lib/trinity/llm/providers/req_llm.ex | 27 + lib/trinity/llm/request.ex | 48 +- lib/trinity/sessions.ex | 17 +- lib/trinity/sessions/prompt.ex | 14 + lib/trinity/sessions/session.ex | 23 +- lib/trinity/telemetry/redaction.ex | 4 + lib/trinity/versions.ex | 19 +- lib/trinity_web/live/gateways_live.ex | 24 +- scripts/dev_gateways.sh | 5 +- test/secrets_scan_test.exs | 8 + test/support/telegram/fake_bot_api.ex | 413 +++++++++++++++ test/support/telegram/markdown_v2.ex | 121 +++++ test/trinity/gateways/approvals_test.exs | 29 ++ test/trinity/gateways/console_test.exs | 4 +- test/trinity/gateways/router_test.exs | 5 +- test/trinity/gateways/supervisor_test.exs | 74 +++ .../gateways/telegram/markdown_test.exs | 202 ++++++++ test/trinity/gateways/telegram_test.exs | 485 ++++++++++++++++++ test/trinity/llm/mapping_test.exs | 46 ++ test/trinity/telemetry/redaction_test.exs | 16 + .../live/gateways_live_telegram_test.exs | 58 +++ 48 files changed, 3189 insertions(+), 64 deletions(-) create mode 100644 lib/trinity/gateways/supervisor.ex create mode 100644 lib/trinity/gateways/telegram.ex create mode 100644 lib/trinity/gateways/telegram/client.ex create mode 100644 lib/trinity/gateways/telegram/markdown.ex create mode 100644 lib/trinity/gateways/telegram/media.ex create mode 100644 lib/trinity/gateways/telegram/offset.ex create mode 100644 lib/trinity/gateways/telegram/outbox.ex create mode 100644 lib/trinity/gateways/telegram/poller.ex create mode 100644 lib/trinity/gateways/telegram/updates.ex create mode 100644 test/support/telegram/fake_bot_api.ex create mode 100644 test/support/telegram/markdown_v2.ex create mode 100644 test/trinity/gateways/supervisor_test.exs create mode 100644 test/trinity/gateways/telegram/markdown_test.exs create mode 100644 test/trinity/gateways/telegram_test.exs create mode 100644 test/trinity_web/live/gateways_live_telegram_test.exs diff --git a/.env.example b/.env.example index 2442c5e..a2bd324 100644 --- a/.env.example +++ b/.env.example @@ -39,6 +39,13 @@ TRINITY_LIVE_MODEL= OLLAMA_BASE_URL=http://localhost:11434/v1 OLLAMA_MODEL= +# ---- Telegram gateway (slice 071), optional --------------------------------------------- +# The bot token from @BotFather. Read through Trinity.Config.secret/1 at each Bot API call and +# never stored. The gateway runs only when it is named: TRINITY_GATEWAYS=telegram. Without the +# token it stays idle and /gateways says so. Leave both empty to keep Telegram off. +TELEGRAM_BOT_TOKEN= +# TRINITY_GATEWAYS=telegram + # ---- App runtime (config/runtime.exs reads these; all have defaults) ------------------ # PORT=4000 # DATABASE_PATH= diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d5f19a..394ee21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,30 @@ evidence for each increment is retained by the maintainers and summarised here. tree, engineering rules consolidated into `docs/03-conventions.md`, and this changelog added as the public record of delivered work. +### `slice/071`: Trinity answers on Telegram + +A Telegram bot is now a channel Trinity can be reached from, alongside the console. It answers in +private chats and in groups where it is mentioned or replied to, streams its answer by editing one +message no more than once a second, converts the model's markdown to Telegram's strict MarkdownV2 +so a half-written answer never fails to send, offers approvals as Approve and Deny buttons within the +channel's existing cap, and passes a photo to a model that declares it can see images. A restarted +bot does not process the same message twice: where it is up to is written to disk before a message +is handled. It is off until named (`TRINITY_GATEWAYS=telegram`) and given a token +(`TELEGRAM_BOT_TOKEN`); the application now starts the gateways it is configured with. + +No Telegram library is used. Both candidates were measured: one cannot be pinned under this +project's version rule, and the other keeps its polling position in memory and cannot be pointed at a +test server. The Bot API is a handful of JSON calls over the HTTP client the project already uses. + +Two defects in the gateway core were found and fixed on the way. The pairing prompt told an unknown +sender the very code it asked them to prove they could read on the desktop, which on a public +platform would let anyone pair; it now says only where the code is. And the short approval id a +channel shows was shared by every request raised in the same minute, so typing it decided whichever +came first; an ambiguous id is now refused, and buttons carry the full id. + +The live checks (a real bot, a real phone, a vision model, a scheduled delivery) are listed for the +maintainer to run; everything else runs in the suite against a fake Bot API with no network. + ## 2026-09-26 ### `slice/128`: the checks that guard attribution now say what they check diff --git a/README.md b/README.md index f53a544..43f8a7d 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,10 @@ enforces it. streaming back as the turn runs, and any approval the turn raises rendered into the same conversation. What a channel may approve is capped below what the desktop may: a `write` at most by default, so an `exec` or a `destructive` request is decided at the machine, and the - refusal is receipted. The console adapter ships (`mix trinity.console`); the platforms are next. + refusal is receipted. The console adapter ships (`mix trinity.console`), and so does Telegram: + a bot that answers in private chats and in groups where it is mentioned, streams its reply by + editing one message, offers approvals as buttons (within the same cap), and passes a photo to a + model that can see. Set `TELEGRAM_BOT_TOKEN` and `TRINITY_GATEWAYS=telegram` to turn it on. - **Delegates.** The assistant can hand a bounded piece of work to a child session with a `delegate` tool: its own context, its own history, and a result handed back as text. The parent's conversation never contains the child's messages, which is the point: @@ -171,10 +174,9 @@ enforces it. curator that marks old memories stale and archives the untouched ones (never deleting) run on the same queues; `/oban` shows the jobs. -Not there yet: the messaging platforms themselves (071 Telegram and 072 Discord, which need a bot -token the project does not yet hold), the native desktop shell and signed releases (the rest of M6), -store-and-forward receipts for disconnected operation (026, waiting on an answer from the external -authority plane). The [Milestones](#milestones) section below sets out the order the remaining work +Not there yet: the second messaging platform (072), the native desktop shell and signed releases +(the rest of M6), store-and-forward receipts for disconnected operation (026, waiting on an answer +from the external authority plane). The [Milestones](#milestones) section below sets out the order the remaining work is being built in. The interface is a local web page; the desktop shell exists as a packaging spike, not a product. diff --git a/VERSIONS.md b/VERSIONS.md index 4cf769e..87fba71 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -94,7 +94,7 @@ never pin a version hex marks as retired or vulnerable. | `stream_data` | ~> 1.4 | ✅ in `mix.lock` | Property-based testing: generated input for `test/property`. Apache-2.0; 1.4.0 measured on hex.pm 2026-09-24, released 2026-07-14. Test only, so it ships in no artifact. Elixir has no focused fuzzing library and property-based testing is what the OpenSSF gold criterion `dynamic_analysis` accepts in its place, which is why this is a dependency rather than a habit. Added at Slice 004. | | `jose` | ~> 1.11 | ✅ in `mix.lock` | JOSE (JWK, JWS, JWKS) over OTP's crypto, for the MCP resource server's token validation and the personal profile's embedded authorization server. MIT; 1.11.12 measured on hex.pm 2026-09-22. Not joken, a layer over it this tree does not need. Added at Slice 062. | | `oban_web` | ~> 2.13 | ✅ in `mix.lock` | The Oban dashboard, a LiveView mounted at /oban (dev, and behind `config :trinity, :oban_web, true` elsewhere). Apache-2.0 on hex.pm since the 2.12 line (2.13.0 measured 2026-09-22); the plan of 2026-09-05 wrote it in when it was commercial. Added at Slice 050. | -| `req` | ~> 0.5 | ✅ in `mix.lock` | HTTP client. | +| `req` | ~> 0.5 | ✅ in `mix.lock` | HTTP client. Since Slice 071 also the Telegram Bot API client, in place of a Telegram library. | | `req_llm` | ~> 1.22 | ✅ in `mix.lock` | Provider layer (streaming, tools, structured output, usage). ⚠️ The pin was `~> 1.10` against a recorded latest of 1.10.0; the real latest was twelve minors ahead. Check event shapes against the current version at Slice 011, not against this file's prose. Added at Slice 011. | | `beam_mcp` | ~> 0.9 | ✅ in `mix.lock` | MCP server core, Apache-2.0, ADR-0007 decision 5 (owner decision 2026-09-08, recorded 2026-09-20). 0.9.0 on hex.pm (2026-09-22): the :server seam on both transports and the core server as a named behaviour, which slice 061's wrapper implements; standing before 1.0.0. Server side only: the client, MRTR and OAuth are Trinity's, above it. Added at Slice 059, bumped at 061. The earlier candidate list (anubis_mcp, fastest_mcp, gen_mcp) is history. | | `jido` | not used (ADR-0009, decided 2026-09-20) | 🔍 not a single package | Measured at the Slice 012 checkpoint and not adopted: the agent runtime duplicates PubSub, Oban and the gateways and adds a second tool executor; the action shape is written in-tree at Slice 020 with `jsv` for its schemas. The row stays so the decision is visible where a reader would look for the package. | @@ -141,7 +141,8 @@ never pin a version hex marks as retired or vulnerable. | `burrito` | ~> 1.6 | ✅ in `mix.lock` | ⚠️ ERTS availability drives the OTP pin, and Slice 000 measured it: only the OTP 28 line is fetchable for macOS and Linux. Corrected 2026-09-05: this row previously read `~> 1.5 / 1.5.0 ✅`; that mark was not measured. Added at Slice 001. | | `ex_tauri` | ~> 0.2 | ✅ in `mix.lock` | ⚠️ Declares `otp_release: "~> 27.0"`, and Slice 000's probe refutes the reason it gives: OTP 28 macOS universal returns 200 and OTP 27 returns 404. Whether it runs on the pinned OTP is Slice 001's first measurement. ⚠️ 439 downloads all-time, so the ADR-0004 fallback matrix carries real weight. | | `nostrum` | ~> 0.10 | 🔍 not yet a dependency | Discord. ⚠️ No release in roughly 18 months. R11's trigger already fires. Check intents and components against the current gateway before Slice 072. | -| `telegex` | **not pinned** | 🔍 not a single package | Telegram. ⚠️ The latest release on hex is a release candidate, roughly two years old, and this file's own rule forbids pinning an `-rc`. Alternative: ex_gram. Slice 071 decides with the measurement. | +| `telegex` | not used (measured at Slice 071, 2026-10-08) | 🔍 not a single package | Telegram. Measured at Slice 071: the latest release on hex is `1.9.0-rc.0` (2024-09-18) and the last stable `1.8.0` (2024-05-28); this file's own rule forbids pinning an `-rc`. Not used: `Trinity.Gateways.Telegram` speaks the Bot API with `req` (slice 071 NOTES, "The library, measured"). The row stays so the decision is visible where a reader would look for the package. | +| `ex_gram` | not used (measured at Slice 071, 2026-10-08) | 🔍 not a single package | Telegram. Measured at Slice 071 on `0.71.0` (2026-10-01): its poller keeps the `getUpdates` offset in process state and starts at `-1`, so a restarted poller reprocesses or skips unconfirmed updates (Slice 071's AC5 is that property); its file download host is a module attribute that cannot be pointed at a test server; Beerware licence; 18,054 lines for what is here a handful of JSON calls. Not used: `Trinity.Gateways.Telegram` speaks the Bot API with `req` and owns its poller. | | `phoenix_streamdown` | not used (measured at Slice 013, 2026-09-20) | 🔍 not a single package | Streaming markdown renderer for LiveView. Measured at Slice 013: 1.0.0-beta.4 (2026-05-03) was still the latest release with no stable behind it, and this file's own rule forbids a pre-release. It is 801 lines over `mdex`, whose own `streaming: true` option completes fragments; the rest is a rendering rule Slice 013 keeps anyway. The row stays so the decision is visible where a reader would look for the package. | | `mdex` | ~> 0.13 | ✅ in `mix.lock` | Markdown renderer for the chat (`TrinityWeb.Markdown`), streaming fragments completed by its `streaming: true` option, raw HTML omitted (`unsafe: false`) and the default sanitizer on top. Added at Slice 013 after the measurement in its NOTES.md: `earmark` 1.4.49 is retired on hex with an open XSS advisory (EEF-CVE-2026-48591), which the gate refuses. ⚠️ A Rust NIF (`mdex_native`): the first in the bundle. Measured at Slice 013 (NOTES finding 13): neither precompiled artifact loads in Burrito's musl ERTS on Linux, so the linux package builds it from source for musl with Zig as the linker (`rustler` below, `scripts/zig-cc-musl`, the three settings in config/config.exs); macOS and Windows load the precompiled artifact. The `--smoke` path prints whether it rendered, and the `package` workflow reads that line on every target. | | `jsv` | ~> 0.23 | ✅ in `mix.lock` | JSON Schema (2020-12) validation of tool arguments in `Trinity.Tools.Schema`, with `cast: false` so a malformed call is refused and never repaired (docs/07). Was transitive through req_llm; direct since Slice 020 because a module of ours calls it (ADR-0009: Trinity's own tool behaviour, jsv for its schemas). | diff --git a/config/config.exs b/config/config.exs index 7e20d42..a742701 100644 --- a/config/config.exs +++ b/config/config.exs @@ -241,3 +241,9 @@ import_config "#{config_env()}.exs" # Slice 070: a scheduled run can be delivered to a gateway conversation; `deliver_to`'s "kind" # names the implementation (slice 050's `Trinity.Scheduler.Delivery.for/1`). config :trinity, :deliveries, %{"gateway" => Trinity.Scheduler.Delivery.Gateway} + +# Slice 071: the gateways this build carries, by module. None runs unless it is named: +# `config :trinity, :gateways, adapters: [...]` names modules, and `TRINITY_GATEWAYS` +# (config/runtime.exs) names adapters from this list by their name ("telegram", "console"), so an +# environment variable never mints an atom. A new platform is a module and a line here. +config :trinity, :gateways, available: [Trinity.Gateways.Console, Trinity.Gateways.Telegram] diff --git a/config/llm.exs b/config/llm.exs index b4d8026..df2c836 100644 --- a/config/llm.exs +++ b/config/llm.exs @@ -7,6 +7,14 @@ import Config # are US dollars per million tokens and are the source of every recorded cost. Free-tier # models carry 0.0. The default model is the OpenRouter one the owner picked; the NVIDIA # endpoint is reached as an OpenAI-compatible base_url. Test config replaces all of this. +# Slice 071: the default OpenRouter model takes images. llm_db's snapshot (the dependency in +# mix.lock) lists `openrouter` / `inclusionai/ling-3.0-flash-vl:free` with input modalities +# ["text", "image", "video"]. `:vision` is declared for that id only: a TRINITY_LIVE_MODEL naming +# another model gets no vision until someone checks that one and says so here. +ling_default = "inclusionai/ling-3.0-flash-vl:free" +ling_model = System.get_env("TRINITY_LIVE_MODEL", ling_default) +ling_vision = if ling_model == ling_default, do: [:vision], else: [] + config :trinity, :llm, default_model: "openrouter:ling", providers: %{req_llm: Trinity.LLM.Providers.ReqLLM}, @@ -15,11 +23,9 @@ config :trinity, :llm, %{ id: "openrouter:ling", provider: :req_llm, - model: - "openrouter:" <> - System.get_env("TRINITY_LIVE_MODEL", "inclusionai/ling-3.0-flash-vl:free"), + model: "openrouter:" <> ling_model, api_key_env: "OPENROUTER_API_KEY", - caps: [:stream, :tools, :json], + caps: [:stream, :tools, :json] ++ ling_vision, price: %{input: 0.0, output: 0.0} }, %{ diff --git a/config/runtime.exs b/config/runtime.exs index a50bc2f..8804e93 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -109,6 +109,21 @@ if config_env() != :test and auth_profile not in [nil, ""] do client_metadata_url: present.("TRINITY_MCP_AUTH_CLIENT_METADATA_URL") end +# Slice 071: which gateways run. `TRINITY_GATEWAYS=telegram` (comma separated) names adapters from +# `config :trinity, :gateways, available:` by name; `Trinity.Gateways.adapters/0` matches them, so +# a name that is not available matches nothing and mints no atom. Unset, no gateway runs. The +# suite configures its own and never reads this. +if config_env() != :test do + case System.get_env("TRINITY_GATEWAYS") do + names when names in [nil, ""] -> + :ok + + names -> + config :trinity, :gateways, + enabled: names |> String.split(",", trim: true) |> Enum.map(&String.trim/1) + end +end + # Slice 013. `TRINITY_FAKE_PROVIDER=1 mix phx.server` runs the chat on the scripted provider: # the registry becomes the fake's two entries and a fresh stream answers with its markdown # demo, so the UI can be exercised and screenshotted with no key and no egress. Development diff --git a/docs/01-architecture.md b/docs/01-architecture.md index 7744b6c..a34b62d 100644 --- a/docs/01-architecture.md +++ b/docs/01-architecture.md @@ -53,8 +53,12 @@ Trinity.Application │ # a dead one dropped), writes the index rows on first read ├── Oban # cron + durable jobs. Slice 050 ├── Trinity.MCP.Supervisor # MCP clients (one per server, lib per 059) + server. Slice 059-062 -├── Trinity.Gateways.Router # one road from a channel to a session. Slice 070 -├── Trinity.Gateways.Console # the in-process adapter; platforms at 071, 072 +├── Trinity.Gateways.Supervisor # Slice 071: the router and the configured adapters, +│ │ # no child when none is configured (until 071 nothing +│ │ # in the application started a gateway) +│ ├── Trinity.Gateways.Router # one road from a channel to a session. Slice 070 +│ ├── Trinity.Gateways.Console # the in-process adapter, when configured. Slice 070 +│ └── Trinity.Gateways.Telegram # outbox and poller (long-polling getUpdates). Slice 071 ├── Trinity.Subagents.Supervisor (DynamicSupervisor) # Slice 080 ├── Trinity.Sandbox.Supervisor # Luerl workers. Slice 110 ├── Trinity.Desktop # ex_tauri bridge (tray, notifications). Slice 100 @@ -92,7 +96,7 @@ without anything failing. | `Trinity.Skills` | SKILL.md parsing, registry, loader, manager, scanner (as built at 040: parser, sources, registry, index, the three tools; at 041: staging, promotion, manager, scanner, diff, learn, `skill_manage` and `learn`) | Repo, Permissions, **Effects**, **Receipts**, Sandbox (as built at 041: Tools, Memory, Permissions, Receipts and LLM; Effects is not a dependency: the promotion is not a tool call, it writes its own effect receipt; Tools never depends on Skills) | | `Trinity.Scheduler` | Oban workers for agent tasks, delivery (as built at 050: the `tasks` and `task_runs` rows, `next_run_at` computed from a cron expression or a one-shot datetime, `Workers.Tick` (the one Cron plugin entry, every minute, enqueuing `Workers.RunTask` for what is due), `RunTask` (one turn in a fresh `origin: "cron"` session, the summary, the delivery), `Delivery` with `Delivery.Desktop`, `Parse.human/2` (a phrase to cron through the model, checked by Oban's parser); Oban itself on the app's repo, Lite on SQLite and Basic on Postgres, queues `agent_tasks`, `memory`, `maintenance`; the observer as `Trinity.Memory.ObserverWorker` and the curator as `Trinity.Memory.Curator` live in Memory) | Sessions, LLM, **Repo** (Gateways at 070) | | `Trinity.MCP` | Client manager, tool bridge, server (as built at 059: the boundary alone, holding the core's version, its JSON depth and its telemetry event names; at 060: `Servers` and `ServerConfig` (the rows), `Supervisor` and `Boot` (one `Client` per enabled row), `Client` with its `Wire` (the outbound request, the headers, the core's decoder and validator) and two transports (`Transport.Stdio`, a child on a Port; `Transport.HTTP`, one POST per request), `Bridge` (the one tool module every MCP tool runs through) and `Client.Auth` (062's seam, a static token at 060); at 061: `Server` (the module above the core, `@behaviour BeamMCP.Server`, handed to both transports through `:server`; `tools/call` answered through the membrane, everything else the core's), `Server.Catalog` (the exported tools, sorted), `Server.Exports` (the configured names, `:catalog` refused), `Server.Session` (the `origin: "mcp"` session with its own persona), `Server.Envelope` and `Server.Replay` (the sealed `requestState` and the nonce table), `Server.Auth.Local` (the static bearer, 062's seam on the server side), `Server.Plug` (the `POST /mcp` endpoint, mounted in `TrinityWeb.Endpoint` ahead of the parsers) and `Server.Stdio` with `mix trinity.mcp.stdio`; at 062: `Trinity.MCP.Auth`, a boundary of its own under `Trinity.MCP` with `deps: []` (the behaviour `authorize/2` answering a `Principal`, never the token; `Config`, `Token`, `JWKS`, `Discovery`, `Scopes`; the profiles `Local`, `External` and `Embedded` with `Embedded.Keys`; the client role `Client` with `Client.Store`), and `AuthHost` in `Trinity.MCP` (the configuration from the tree's paths and the authority selection, the personal profile's boot, the refusal receipts, the client role's flow for the page); `Server.Auth.Local` is gone, `Server.Plug` authorizes ahead of the transport and answers `401` with a challenge, `Server` checks scope before the gate) | Tools, **Effects**, **Permissions**, Memory (as built at 059: a top-level boundary, like `Trinity.Smoke`, with `deps: [Trinity, BeamMCP.JSON]`; the boundary compiler checks every call into the `beam_mcp` application and this boundary alone lists its modules; at 060 the deps are `[Trinity, BeamMCP.JSON, BeamMCP.Schema]`, Trinity's tools, permissions and receipts reached through `Trinity`'s exports, and `Trinity.Application` and `TrinityWeb` list `Trinity.MCP`; at 061 the core's `Server`, `Catalog`, `ToolSpec` and both transports are listed too, and `Trinity` exports `Tools.Registry` and `Effects.Runner` to it) | -| `Trinity.Gateways` | Adapter behaviour, router, allowlists, pairing (as built at 070: `Adapter` (the behaviour: `child_spec/1`, `capabilities/0`, `deliver/2`, `format/2`, `render_approval/2`, plus `name/1` deriving the row's adapter name from the module), `Console` (the in-process adapter the tests and `mix trinity.console` talk to), `Format` (chunking to the channel's limit, re-opening a split code fence, markdown reduced to text where a channel renders none, the default approval rendering), `Identity` and `Identities` (the `gateway_identities` rows, the pairing code and the allowlist), `Router` (one process: admission, then the token bucket, then commands, then the session; it subscribes to `session:` and `approvals:` per conversation and coalesces deltas into an edited message), `Commands` (the slash registry, including `/attach`, `/approve` and `/deny`), `Cap` (the channel trust cap) and `Receipts` (the capped-decision receipt)) | Sessions, **Permissions**, PubSub | +| `Trinity.Gateways` | Adapter behaviour, router, allowlists, pairing (as built at 070: `Adapter` (the behaviour: `child_spec/1`, `capabilities/0`, `deliver/2`, `format/2`, `render_approval/2`, plus `name/1` deriving the row's adapter name from the module), `Console` (the in-process adapter the tests and `mix trinity.console` talk to), `Format` (chunking to the channel's limit, re-opening a split code fence, markdown reduced to text where a channel renders none, the default approval rendering), `Identity` and `Identities` (the `gateway_identities` rows, the pairing code and the allowlist), `Router` (one process: admission, then the token bucket, then commands, then the session; it subscribes to `session:` and `approvals:` per conversation and coalesces deltas into an edited message), `Commands` (the slash registry, including `/attach`, `/approve` and `/deny`), `Cap` (the channel trust cap) and `Receipts` (the capped-decision receipt); at 071: `Supervisor` (the router and the adapters in force, started by the application, nothing when none is configured; `TRINITY_GATEWAYS` names them from the `available:` list), the optional `status/0` callback and the `{:message, text, buttons}` outbound, attachments the router fetches only after admission, and `Telegram` (`Outbox`, `Poller`, `Updates`, `Markdown` to MarkdownV2, `Media`, `Offset`, `Client` over Req)) | Sessions, **Permissions**, PubSub | | `Trinity.Subagents` | Delegation, result collection (as built at 080: `delegate/3` and `delegate_many/3` creating child sessions with `origin: "subagent"` and `parent_id` set, one turn each under a monitor, the awaiting shape reused from slice 050's `RunTask` including its drain of the start `:idle`; budgets of turns, tokens and wall clock; `restarts` defaulting to **0**, because replaying a turn replays arbitrary tool calls and a `:session` or `:always` approval would cover the replay silently; `cancel_subtree/1` depth first; `running?/1` asked of the process rather than of `SessionRow.status`, which is the session lifecycle and stays `active` after a subagent finishes; `short_id/1` taking the **tail** of a UUIDv7, whose head is shared between children created in the same second) | Sessions | | `Trinity.Sandbox` | Luerl runners, resource limits | none | | `Trinity.Desktop` | ex_tauri bridge | PubSub | diff --git a/docs/02-tech-stack.md b/docs/02-tech-stack.md index 46c0ab8..18eeca8 100644 --- a/docs/02-tech-stack.md +++ b/docs/02-tech-stack.md @@ -21,7 +21,7 @@ Versions live in `VERSIONS.md`. This file explains *why* each choice was made an | Config validation | nimble_options | Behaviour opts validated with docs generated | none | | Desktop shell | ex_tauri (Tauri 2 + Burrito sidecar) | Modern webview, tray, notifications, updater, signing plumbing; small footprint | Windows unsupported by ex_tauri → elixir-desktop or plain Tauri sidecar (Slice 001 decides; ADR-0004) | | Packaging | Burrito | Single binary with ERTS | none | -| Gateways | Telegex (Telegram), Nostrum (Discord) | Active, supervised | none | +| Gateways | Telegram: the Bot API over Req, no library (Telegex and ex_gram measured at 071 and not used; VERSIONS.md says why). Nostrum (Discord) | Active, supervised | none | | Markdown streaming | phoenix_streamdown | LLM-optimised; freezes completed blocks | Verify in 013; fallback to earmark + chunk buffering | | Testing | ExUnit, Mox, LiveViewTest (lazy_html) | Standard | none | | Quality | credo, mix_audit, sobelow, ex_doc, Elixir type checker | Gate | none | diff --git a/docs/05-data-model.md b/docs/05-data-model.md index 51676f5..fc6d49c 100644 --- a/docs/05-data-model.md +++ b/docs/05-data-model.md @@ -37,6 +37,7 @@ Slice 033 adds `project_root` (string, nullable): the directory the session's to | role | string | "system" \| "user" \| "assistant" \| "tool" | | content | text | plain text or JSON for tool payloads | | parts | map | structured content parts (text, tool_call, tool_result, image ref). Every part carries `origin`, `source_ref`, `digest` and `taint ∈ {trusted, untrusted, blocked}`; summaries inherit the maximum taint of their inputs (Slice 022) | +| (images, Slice 071) | | a `user` row that came with images carries `parts.images`, a list of `path` (under the data directory, named by digest), `media_type`, `digest` (`sha256:`), `bytes` and `origin` (`gateway:telegram`). The model receives them only if its registry entry declares `:vision` | | tool_call_id | string, nullable | the assistant row's call id a `tool` row answers (Slice 012); its `parts` carry `tool`, `ok`, `tool_result` (`content`, `truncated`, `meta`, `artifacts`, or `error`) and `tool_definition_digest`, and the assistant row's `provider_meta.tool_surface` maps every declared tool name to its digest (Slice 020) | | (compaction rows, Slice 023) | | a `system` row whose `parts.compaction` carries `from_seq`, `to_seq`, `rows`, `digests` (of the summarised rows' content parts), `summary`, `open_threads`, `decisions`, `facts` and `previous` (the earlier compaction's id); `parts.taint` the maximum of the inputs'. Nothing it covers is edited or deleted: the prompt builder renders the newest compaction into the system prompt and drops the rows it covers from the list. A fork (past the hard threshold) is a child session with `parent_id` whose first row is the compaction and whose second is the user's message; the parent's last row carries `parts.forked_to` | | usage | map, nullable | prompt/completion tokens, cost | diff --git a/docs/07-security-model.md b/docs/07-security-model.md index 678ba2c..6f85c0b 100644 --- a/docs/07-security-model.md +++ b/docs/07-security-model.md @@ -319,6 +319,52 @@ regulated effect. hole shaped exactly like the thing the cap exists for. `Trinity.Gateways.Cap` applies the ceiling to every adapter, `Console` included, and an unknown tier is refused rather than waved through. +- **Corrected at 071: the pairing prompt no longer carries the code.** Until slice 071 the prompt an + unknown sender received named the code it asked them to send back, so the "shown on `/gateways` + and nowhere else" above was not true of the code: anyone who could write to a channel could pair + by echoing the prompt. On the console that channel is the machine's own terminal; on a messaging + platform it is anyone who finds the bot. The prompt now says where the code is and never what it + is, and 070's test asserts the opposite of what it asserted before. +- **Corrected at 071: an ambiguous approval id is refused.** Approval ids are UUIDv7, so the + eight-character short form a channel shows is the same for every request raised within the same + 65 seconds, and `/approve ` decided whichever pending request came first. A prefix that + names more than one pending request is now refused and neither is decided; buttons carry the full + id. + +### Telegram (Slice 071, as built) + +- **The token** is `TELEGRAM_BOT_TOKEN`, read through `Trinity.Config.secret/1` at each Bot API call + and held in no process's state. The Bot API puts it in the URL path, so nothing the adapter returns + or logs is built from a URL or an exception's message: errors carry Telegram's own code and + description or the transport's reason atom. The log filter (`Trinity.Telemetry.Redaction`) masks + the token's shape as a last line, and the secret scanner refuses one in the tree. +- **Inbound is the router's, as for every adapter.** Private chats are read; in a group, only a + message that mentions the bot, replies to it, or is a command addressed to it. Bots and channels + are ignored. Each sender is an identity of their own and pairs for themselves. +- **Images are fetched only for an admitted sender.** The adapter hands the router a function that + downloads the image; the router calls it after the pairing check and the rate limit, and never for + a command, so a stranger's photo is not downloaded. An image is refused above a size limit (10 MB by + default) and when its first bytes are not the format it claims; a stored image is named by its + SHA-256 digest under the data directory, and the user row records its path, media type, digest + and origin. It reaches the model as an image only when the model's registry entry declares + `:vision`; otherwise the model is told an image was attached and is not sent it. +- **Approval buttons are text.** A button carries `/approve ` or `/deny ` with the request's + full id, and a press is routed as that text from the person who pressed it: identity, rate limit, + channel cap and gate all apply as they do to a typed command, so a forged press can say nothing its + sender could not type. A request above the channel's ceiling is rendered with no buttons, only where + to decide it. Callback data that is not one of the two commands is acknowledged and ignored. +- **Updates are processed at most once.** The `getUpdates` offset is written to the data directory + before an update is routed, so a restarted poller never processes one twice; the cost is that a + crash between the write and the routing loses that one message. +- **What crosses to Telegram.** The conversation's text in both directions (the person's messages, + Trinity's answers, command answers), approval requests (tool name, risk tier and a summary of the + arguments, each value cut to 40 characters), the pairing prompt (which carries no code), and scheduled + run summaries addressed to a chat. Images go one way only, from Telegram to this machine. Nothing is + hashed on the way out: Telegram is the channel, and a person reads it there. This is why a + regulated node refuses the adapter unless `TRINITY_REGULATED_GATEWAYS` names it. +- **Not built:** webhook mode (an inbound HTTPS route whose only credential is a header, worth a + review of its own), sending images, voice notes and stickers. + ## Effects and receipts (Slice 024, as built) The runner in force is `Trinity.Effects.Runner`. For every validated call it asks the gate once and writes a diff --git a/lib/mix/tasks/trinity.secrets.scan.ex b/lib/mix/tasks/trinity.secrets.scan.ex index c67213d..d96b9fd 100644 --- a/lib/mix/tasks/trinity.secrets.scan.ex +++ b/lib/mix/tasks/trinity.secrets.scan.ex @@ -16,6 +16,8 @@ defmodule Mix.Tasks.Trinity.Secrets.Scan do {"GitHub token", ~r/\bghp_[A-Za-z0-9]{36}\b/}, {"Slack token", ~r/\bxox[baprs]-[A-Za-z0-9-]{10,}\b/}, {"private key block", ~r/-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/}, + # Slice 071: `:AA<33 more>`, the shape BotFather issues. + {"Telegram bot token", ~r/\b[0-9]{8,10}:AA[A-Za-z0-9_-]{33}\b/}, {"generic long secret assignment", ~r/(?i)\b(?:api[_-]?key|secret[_-]?key|access[_-]?token)\b\s*[:=]\s*["'][A-Za-z0-9\/+_-]{24,}["']/} ] diff --git a/lib/trinity/application.ex b/lib/trinity/application.ex index 59e47f3..ce1d9b3 100644 --- a/lib/trinity/application.ex +++ b/lib/trinity/application.ex @@ -88,7 +88,11 @@ defmodule Trinity.Application do Trinity.Sessions.Supervisor, # Slice 050: Oban after the sessions its workers drive (a run is a turn in a session), # the engine chosen by the adapter (config.exs). - {Oban, Application.fetch_env!(:trinity, Oban)} + {Oban, Application.fetch_env!(:trinity, Oban)}, + # Slice 071: the gateways in force (the router and each configured adapter), after the + # sessions they route to and the scheduler that delivers to them; no child when none is + # configured. + Trinity.Gateways.Supervisor ] ++ Trinity.Smoke.probe(Trinity.Smoke.argv()) ++ [ diff --git a/lib/trinity/gateways.ex b/lib/trinity/gateways.ex index 10fcd82..2ee1580 100644 --- a/lib/trinity/gateways.ex +++ b/lib/trinity/gateways.ex @@ -9,14 +9,30 @@ defmodule Trinity.Gateways do the permission gate's answer, as it is for the desktop, and what an approval arriving from a channel may authorise is capped besides (`Cap`, docs/07 "Gateways"). - Adapters are configured, not compiled in: `config :trinity, :gateways, adapters: [Module, …]`. - `Console` ships here and is what the tests and `mix trinity.console` talk to. + Adapters are configured, not compiled in: `config :trinity, :gateways, adapters: [Module, …]`, + or by name from the `available:` list (`TRINITY_GATEWAYS=telegram`, slice 071). `Console` ships + here and is what the tests and `mix trinity.console` talk to; `Telegram` is the first platform. + `Trinity.Gateways.Supervisor` runs the router and the adapters in force, and nothing when there + are none. """ use Boundary, deps: [Trinity, Trinity.Sessions], - exports: [Adapter, Cap, Console, Format, Identities, Identity, Router] + exports: [Adapter, Cap, Console, Format, Identities, Identity, Router, Supervisor, Telegram] - @doc "The adapter modules in force." + @doc """ + The adapter modules in force: those named in `adapters:`, then those from `available:` whose + name (`Trinity.Gateways.Adapter.name/1`) is in `enabled:`, each once. + """ @spec adapters() :: [module()] - def adapters, do: Application.get_env(:trinity, :gateways, []) |> Keyword.get(:adapters, []) + def adapters do + config = Application.get_env(:trinity, :gateways, []) + enabled = Keyword.get(config, :enabled, []) + + named = + config + |> Keyword.get(:available, []) + |> Enum.filter(&(Trinity.Gateways.Adapter.name(&1) in enabled)) + + Enum.uniq(Keyword.get(config, :adapters, []) ++ named) + end end diff --git a/lib/trinity/gateways/adapter.ex b/lib/trinity/gateways/adapter.ex index 32f9985..6e00d03 100644 --- a/lib/trinity/gateways/adapter.ex +++ b/lib/trinity/gateways/adapter.ex @@ -34,12 +34,25 @@ defmodule Trinity.Gateways.Adapter do @typedoc """ What the router asks an adapter to do. `{:edit, ref, text}` carries the reference the adapter returned when it sent the message being replaced; an adapter without `edits` never receives one. + `{:message, text, buttons}` (slice 071) is a message with buttons, each a label and the command + text a press sends back; only an adapter whose `render_approval/2` returned one receives it. """ @type outbound :: {:message, String.t()} + | {:message, String.t(), [button()]} | {:edit, reference_id(), String.t()} | {:typing, boolean()} + @typedoc """ + A button: what it says, and the text a press sends back through the router as if typed + (slice 071). A press is routed like any other inbound text, so a button can do nothing its + presser could not have typed. + """ + @type button :: {label :: String.t(), command :: String.t()} + + @typedoc "What `/gateways` shows beside an adapter that reports more than whether it runs." + @type status :: %{state: atom(), detail: String.t()} + @typedoc "Whatever the adapter uses to name a message it sent, opaque to the router." @type reference_id :: term() @@ -78,4 +91,12 @@ defmodule Trinity.Gateways.Adapter do channel without buttons still has to be able to answer. """ @callback render_approval(Approval.t(), capabilities()) :: outbound() + + @doc """ + What the adapter is doing, for `/gateways` (slice 071): connected as whom, idle and why, or the + last error. Optional; an adapter without it is shown as running or not. + """ + @callback status() :: status() + + @optional_callbacks status: 0 end diff --git a/lib/trinity/gateways/commands.ex b/lib/trinity/gateways/commands.ex index faa4233..27484f6 100644 --- a/lib/trinity/gateways/commands.ex +++ b/lib/trinity/gateways/commands.ex @@ -98,11 +98,15 @@ defmodule Trinity.Gateways.Commands do defp decide(command, argument, message) do case find_pending(argument) do - nil -> + [] -> "No pending approval starts with #{argument}." - approval -> + [approval] -> answer(command, approval, message) + + several -> + "#{argument} matches #{length(several)} pending requests, so neither is decided. " <> + "Use the buttons, or more of the id." end end @@ -133,8 +137,11 @@ defmodule Trinity.Gateways.Commands do defp decider(message), do: "gateway:" <> Adapter.name(message.adapter) <> ":" <> message.external_user_id + # Every pending request the prefix names, so an ambiguous one is refused rather than resolved to + # whichever came first (slice 071, finding F2: ids are UUIDv7, time-ordered, and two requests + # raised within the same 65 seconds share their first eight characters). defp find_pending(argument) do - Enum.find(Permissions.pending(:all), &String.starts_with?(&1.id, argument)) + Enum.filter(Permissions.pending(:all), &String.starts_with?(&1.id, argument)) end # `/new` forgets the binding; the next message creates a session as a first message does. diff --git a/lib/trinity/gateways/router.ex b/lib/trinity/gateways/router.ex index a801b57..983d345 100644 --- a/lib/trinity/gateways/router.ex +++ b/lib/trinity/gateways/router.ex @@ -30,13 +30,26 @@ defmodule Trinity.Gateways.Router do @flush_ms 700 @default_rate [capacity: 20, per_minute: 20] - @typedoc "What an adapter hands in: who said what, where." + @typedoc "What an adapter hands in: who said what, where, and what came with it." @type inbound :: %{ adapter: module(), conversation: Adapter.conversation(), external_user_id: String.t(), text: String.t(), - display_name: String.t() | nil + display_name: String.t() | nil, + attachments: [attachment()] + } + + @typedoc """ + Something sent with a message (slice 071): an image the adapter can fetch. The router calls + `fetch` only after the sender is admitted and inside the rate limit, and never for a command, so + nothing a stranger sends is downloaded (docs/07: nothing of an unknown sender's message is read + but the code it might be). A fetch answers the image part a session row stores. + """ + @type attachment :: %{ + required(:kind) => :image, + required(:fetch) => (-> {:ok, map()} | {:error, term()}), + optional(atom()) => term() } ## The API an adapter uses @@ -47,8 +60,9 @@ defmodule Trinity.Gateways.Router do @doc """ One inbound message. Answers once the message has been placed (the reply arrives on the channel - as the turn streams), or with why it was refused: `{:error, :pending}` when a pairing code was - shown instead, `{:error, :revoked}`, `{:error, :rate_limited}`. + as the turn streams), or with why it was refused: `{:error, :pending}` when a pairing prompt was + shown instead, `{:error, :revoked}`, `{:error, :rate_limited}`. Options: `:display_name`, and + `:attachments` (slice 071), fetched only once the message is admitted. """ @spec inbound(module(), Adapter.conversation(), String.t(), String.t(), keyword()) :: {:ok, :placed | :command | :paired} | {:error, term()} @@ -58,7 +72,8 @@ defmodule Trinity.Gateways.Router do conversation: conversation, external_user_id: external_user_id, text: text, - display_name: Keyword.get(opts, :display_name) + display_name: Keyword.get(opts, :display_name), + attachments: Keyword.get(opts, :attachments, []) } GenServer.call(__MODULE__, {:inbound, message}, 30_000) @@ -162,10 +177,14 @@ defmodule Trinity.Gateways.Router do end end + # The prompt says where the code is and never what it is: the code is shown on `/gateways` and + # nowhere else, which is the whole of the pairing proof (docs/07). It said the code until slice + # 071 (finding F1), which on a public platform let anyone who found the bot pair by echoing it. defp prompt_for(identity) do """ - Trinity does not know you yet. Open the desktop app, go to Settings → Gateways, and you will - see the code #{identity.code}. Send that code here to pair. It lasts #{div(Identities.code_ttl_s(), 60)} minutes. + Trinity does not know you yet. Open the desktop app and go to Gateways: a \ + #{String.length(identity.code)} character code is waiting there for this account. Send that \ + code here to pair. It lasts #{div(Identities.code_ttl_s(), 60)} minutes. """ |> String.trim() end @@ -197,8 +216,9 @@ defmodule Trinity.Gateways.Router do defp dispatch(message, state) do {session_id, state} = session_for(message, state) + {images, state} = fetch_attachments(message, state) - case Sessions.send_user_message(session_id, message.text) do + case Sessions.send_user_message(session_id, message.text, images: images) do {:ok, _} -> {:reply, {:ok, :placed}, state} @@ -210,6 +230,36 @@ defmodule Trinity.Gateways.Router do end end + # The sender is admitted and inside the rate limit by the time this runs, which is the point: + # an attachment is read for someone who has been let in and for no one else. A fetch that fails + # or raises costs its image, not the message: the channel is told and the text still goes. + defp fetch_attachments(%{attachments: []}, state), do: {[], state} + + defp fetch_attachments(message, state) do + Enum.reduce(message.attachments, {[], state}, fn attachment, {images, state} -> + case safe_fetch(attachment) do + {:ok, image} -> + {images ++ [image], state} + + {:error, reason} -> + Logger.warning("gateway: attachment not fetched: #{inspect(reason)}") + {images, say(message, "That image could not be read (#{describe(reason)}).", state)} + end + end) + end + + defp safe_fetch(%{fetch: fetch}) when is_function(fetch, 0) do + fetch.() + rescue + error -> {:error, error.__struct__} + end + + defp safe_fetch(_attachment), do: {:error, :not_an_attachment} + + defp describe({:too_large, bytes}), do: "it is larger than this channel accepts: #{bytes} bytes" + defp describe({:not_an_image, type}), do: "its bytes are not a #{type}" + defp describe(_reason), do: "the platform did not hand it over" + ## Sessions and binding defp session_for(message, state) do @@ -378,14 +428,20 @@ defmodule Trinity.Gateways.Router do conv_key -> conv = Map.fetch!(state.conversations, conv_key) - {:message, text} = conv.adapter.render_approval(approval, conv.adapter.capabilities()) + rendered = conv.adapter.render_approval(approval, conv.adapter.capabilities()) + + # Buttons (slice 071) are offered only for a tier this channel may answer; above it the + # request is said in words, with where to decide it, and nothing to press. + outbound = + case {Cap.allows?(conv.adapter, approval.risk), rendered} do + {true, rendered} -> + rendered - text = - if Cap.allows?(conv.adapter, approval.risk), - do: text, - else: text <> "\n\n" <> Cap.refusal(conv.adapter, approval.risk) + {false, rendered} -> + {:message, elem(rendered, 1) <> "\n\n" <> Cap.refusal(conv.adapter, approval.risk)} + end - _ = conv.adapter.deliver(conv.conversation, {:message, text}) + _ = conv.adapter.deliver(conv.conversation, outbound) state end end diff --git a/lib/trinity/gateways/supervisor.ex b/lib/trinity/gateways/supervisor.ex new file mode 100644 index 0000000..071b9a7 --- /dev/null +++ b/lib/trinity/gateways/supervisor.ex @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Gateways.Supervisor do + @moduledoc """ + The gateways in the application's tree (slice 071): the router, then each adapter in force + (`Trinity.Gateways.adapters/0`), one for one. + + Until this slice nothing in the application started a gateway; 070's router and console ran + under the suite, `mix trinity.console` and a script. A platform adapter has to run in the + application itself, so this is where it does. With no adapter configured it starts no child at + all, so a desktop that uses no gateway runs exactly what it ran before, and the suite (which + starts its own router per test) is unaffected. + + The router comes first because an adapter's first act on an inbound message is to call it. + """ + use Supervisor + + alias Trinity.Gateways + alias Trinity.Gateways.Router + + @doc "Starts the gateways' supervisor." + @spec start_link(keyword()) :: Supervisor.on_start() + def start_link(opts \\ []), do: Supervisor.start_link(__MODULE__, opts, name: __MODULE__) + + @impl Supervisor + def init(_opts), do: Supervisor.init(children(Gateways.adapters()), strategy: :one_for_one) + + @doc "The children for a list of adapters: none for none, else the router and each adapter." + @spec children([module()]) :: [Supervisor.child_spec() | {module(), term()} | module()] + def children([]), do: [] + def children(adapters), do: [Router | Enum.map(adapters, &{&1, []})] +end diff --git a/lib/trinity/gateways/telegram.ex b/lib/trinity/gateways/telegram.ex new file mode 100644 index 0000000..4aeb1bb --- /dev/null +++ b/lib/trinity/gateways/telegram.ex @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Gateways.Telegram do + @moduledoc """ + Trinity reached from Telegram (slice 071): a bot, long-polling the Bot API, in private chats and + in groups where it is addressed. + + It is a `Trinity.Gateways.Adapter` and holds that behaviour's rule: it carries text (and the + images a person sends) between Telegram and `Trinity.Gateways.Router`, and nothing else. Three + processes under this supervisor: + + - `Telegram.Outbox` sends: MarkdownV2 escaping at the wire, edits throttled to one a second per + message, the typing indicator, and the status `/gateways` shows; + - `Telegram.Poller` receives: long-polling `getUpdates`, the offset persisted before an update is + routed so a restart never processes one twice; + - `Telegram.Updates` (a module, run by the poller) decides whether the bot was spoken to and hands + the router the text, the sender and any image to fetch. + + **Configuration.** The token is `TELEGRAM_BOT_TOKEN`, read through `Trinity.Config.secret/1` at + every call and never stored. The rest is `config :trinity, :telegram`: `base_url` (default + `https://api.telegram.org`; the suite points it at a fake Bot API), `state_dir` (default + `/gateways/telegram`, where the offset and received images live), `poll_timeout_s` + (50) and `max_image_bytes` (10 MB). The adapter is turned on like any other, by naming it in + `config :trinity, :gateways, adapters:` or `TRINITY_GATEWAYS=telegram`. + + **Approvals** carry an inline keyboard whose buttons send `/approve ` and `/deny ` with + the request's full id. A request above this channel's ceiling (`Trinity.Gateways.Cap`, `:write` by + default) gets no buttons, only the router's "decide it on the desktop". + + **Not here:** webhook mode, sending images, voice notes and stickers (NOTES, decisions 8 and 9). + """ + @behaviour Trinity.Gateways.Adapter + + use Supervisor + + alias Trinity.Gateways.{Cap, Format} + alias Trinity.Gateways.Telegram.{Markdown, Outbox, Poller} + alias Trinity.Permissions.Approval + + @capabilities %{markdown: true, images: true, buttons: true, edits: true, max_length: 4096} + @default_poll_timeout_s 50 + @default_max_image_bytes 10 * 1024 * 1024 + + ## The process + + @impl Trinity.Gateways.Adapter + def child_spec(opts) do + %{id: __MODULE__, start: {__MODULE__, :start_link, [opts]}, type: :supervisor} + end + + @doc "Starts the adapter: its outbox and its poller." + @spec start_link(keyword()) :: Supervisor.on_start() + def start_link(opts \\ []), do: Supervisor.start_link(__MODULE__, opts, name: __MODULE__) + + @impl Supervisor + def init(opts), do: Supervisor.init([{Outbox, opts}, {Poller, opts}], strategy: :one_for_one) + + ## The behaviour + + @impl Trinity.Gateways.Adapter + def capabilities, do: @capabilities + + @impl Trinity.Gateways.Adapter + def deliver(conversation, outbound), do: Outbox.deliver(conversation, outbound) + + @doc """ + An answer split into chunks that are each one Telegram message: chunked in markdown (070's + `Format.chunk/2`, which keeps a code fence closed and reopened across a split), then any chunk + whose MarkdownV2 form is longer than `max_length` UTF-16 code units split again. Escaping can + double a text's length and an emoji counts two, so the limit is checked after conversion, not + guessed before it. The chunks stay markdown: `deliver/2` converts at the wire. + """ + @impl Trinity.Gateways.Adapter + def format(text, capabilities) when is_binary(text) do + text + |> Format.chunk(capabilities.max_length) + |> Enum.flat_map(&fit(&1, capabilities.max_length)) + end + + defp fit(chunk, max) do + cond do + Markdown.utf16_length(Markdown.to_markdown_v2(chunk)) <= max -> [chunk] + String.length(chunk) <= 1 -> [chunk] + true -> chunk |> Format.chunk(div(String.length(chunk), 2)) |> Enum.flat_map(&fit(&1, max)) + end + end + + @doc """ + An approval as a message with two buttons, when this channel may answer its tier; the text + carries the typed commands too, so it can be answered without them. + """ + @impl Trinity.Gateways.Adapter + def render_approval(%Approval{} = approval, capabilities) do + {:message, text} = Format.render_approval(approval, capabilities) + + if capabilities.buttons and Cap.allows?(__MODULE__, approval.risk) do + {:message, text, + [{"Approve", "/approve " <> approval.id}, {"Deny", "/deny " <> approval.id}]} + else + {:message, text} + end + end + + @doc "What `/gateways` shows beside the channel: polling as which bot, idle, or the last error." + @impl Trinity.Gateways.Adapter + @spec status() :: %{state: atom(), detail: String.t()} + def status do + if Process.whereis(Outbox), + do: Outbox.status(), + else: %{state: :stopped, detail: "not started"} + end + + ## Configuration + + @doc "Where the offset and received images live." + @spec state_dir() :: Path.t() + def state_dir do + case Keyword.get(config(), :state_dir) do + nil -> Path.join([Trinity.Paths.data_dir(), "gateways", "telegram"]) + dir -> dir + end + end + + @doc "How long one `getUpdates` is held open, in seconds." + @spec poll_timeout_s() :: non_neg_integer() + def poll_timeout_s, do: Keyword.get(config(), :poll_timeout_s, @default_poll_timeout_s) + + @doc "The largest image accepted, in bytes." + @spec max_image_bytes() :: pos_integer() + def max_image_bytes, do: Keyword.get(config(), :max_image_bytes, @default_max_image_bytes) + + defp config, do: Application.get_env(:trinity, :telegram, []) +end diff --git a/lib/trinity/gateways/telegram/client.ex b/lib/trinity/gateways/telegram/client.ex new file mode 100644 index 0000000..1b4a428 --- /dev/null +++ b/lib/trinity/gateways/telegram/client.ex @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Gateways.Telegram.Client do + @moduledoc """ + The Telegram Bot API, over `Req` (slice 071). No Telegram library: NOTES records the measurement + that decided it (the one candidate that can be pinned keeps its polling offset in memory and + hard-codes its file host, and the other cannot be pinned at all). + + **The token.** It is read from `TELEGRAM_BOT_TOKEN` through `Trinity.Config.secret/1` at every + call, as a provider key is, and never held in a process's state. The Bot API puts it in the URL + path (`/bot/`), which is where a token leaks from: a transport error that prints + its request, a log line that prints a URL. So nothing this module returns or logs is built from + the URL or from an exception's message: a failure is `{:error, {:telegram, code, description}}` + (Telegram's own words, which do not echo the token), `{:error, {:transport, reason}}` with the + transport's reason atom, or `{:error, {:http, status}}`. Req's retry step is off, because it logs; + the poller has its own backoff. + + **The host.** `config :trinity, :telegram, base_url:` (default `https://api.telegram.org`), so + the suite talks to a fake Bot API on a loopback port and never to the network. + """ + + @default_base_url "https://api.telegram.org" + @token_env "TELEGRAM_BOT_TOKEN" + @call_timeout_ms 15_000 + + @typedoc "Why a call did not succeed. None of these carries the token or a URL." + @type error :: + {:missing_secret, String.t()} + | {:telegram, integer(), String.t()} + | {:transport, atom() | String.t()} + | {:http, integer()} + | {:too_large, non_neg_integer()} + + @doc "The environment variable the token is read from." + @spec token_env() :: String.t() + def token_env, do: @token_env + + @doc "Whether a token is configured, without reading it into anything that outlives the call." + @spec configured?() :: boolean() + def configured?, do: match?({:ok, _}, Trinity.Config.secret(@token_env)) + + @doc "The Bot API host in force." + @spec base_url() :: String.t() + def base_url, + do: + :trinity |> Application.get_env(:telegram, []) |> Keyword.get(:base_url, @default_base_url) + + @doc """ + Calls a Bot API method with JSON parameters and answers its `result`. `:receive_timeout` in + `opts` lengthens the wait, which a long poll needs. + """ + @spec call(String.t(), map(), keyword()) :: {:ok, term()} | {:error, error()} + def call(method, params \\ %{}, opts \\ []) when is_binary(method) and is_map(params) do + with {:ok, token} <- Trinity.Config.secret(@token_env) do + [ + method: :post, + url: base_url() <> "/bot" <> token <> "/" <> method, + json: params, + receive_timeout: Keyword.get(opts, :receive_timeout, @call_timeout_ms), + retry: false + ] + |> Req.new() + |> Req.request() + |> answer() + end + end + + @doc """ + Downloads a file `getFile` named, refusing one larger than `max_bytes` (by the size Telegram + reported and again by the bytes that arrived). + """ + @spec download(String.t(), pos_integer()) :: {:ok, binary()} | {:error, error()} + def download(file_path, max_bytes) when is_binary(file_path) do + with {:ok, token} <- Trinity.Config.secret(@token_env) do + [ + method: :get, + url: base_url() <> "/file/bot" <> token <> "/" <> file_path, + receive_timeout: @call_timeout_ms, + retry: false, + decode_body: false + ] + |> Req.new() + |> Req.request() + |> case do + {:ok, %Req.Response{status: 200, body: body}} when byte_size(body) <= max_bytes -> + {:ok, body} + + {:ok, %Req.Response{status: 200, body: body}} -> + {:error, {:too_large, byte_size(body)}} + + other -> + answer(other) + end + end + end + + defp answer({:ok, %Req.Response{body: %{"ok" => true, "result" => result}}}), do: {:ok, result} + + defp answer({:ok, %Req.Response{body: %{"ok" => false} = body}}) do + {:error, + {:telegram, Map.get(body, "error_code", 0), Map.get(body, "description", "no description")}} + end + + defp answer({:ok, %Req.Response{status: status}}), do: {:error, {:http, status}} + + # The reason atom only: an exception's message may be built from the request, and the request + # carries the token in its path. + defp answer({:error, %{reason: reason}}) when is_atom(reason), + do: {:error, {:transport, reason}} + + defp answer({:error, %{__struct__: struct}}), + do: {:error, {:transport, struct |> Module.split() |> Enum.join(".")}} +end diff --git a/lib/trinity/gateways/telegram/markdown.ex b/lib/trinity/gateways/telegram/markdown.ex new file mode 100644 index 0000000..8a380a9 --- /dev/null +++ b/lib/trinity/gateways/telegram/markdown.ex @@ -0,0 +1,308 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Gateways.Telegram.Markdown do + @moduledoc """ + The markdown a model writes, as Telegram's MarkdownV2 (slice 071). + + MarkdownV2 is strict where model markdown is loose: eighteen punctuation characters must be + escaped wherever they are not markup, an entity left open is a parse error, and a parse error is + the whole message refused (`400 Bad Request: can't parse entities`). A streamed answer is + half-written markdown by construction, so the rule this module holds is that **its output parses + whatever its input is**: an entity is emitted only when both its delimiters are present and its + content is not empty, and everything else is escaped as text. A half-streamed `**bol` arrives as + the literal characters, and becomes bold on the edit that closes it. + + What is translated: fenced code (a `pre` entity, its language kept when it is a plain word), + inline code, `**bold**` and `__bold__`, `*italic*` and `_italic_` (an underscore inside a word, + as in `snake_case`, is a character and not markup), `~~strike~~`, `[text](url)` for `http`, + `https`, `tg` and `mailto` URLs (anything else stays text), headings (bold), block quotes, list + markers (a bullet, or the number with its dot escaped) and rules. Inside `pre` and `code` only + `` ` `` and `\\` are escaped; inside a link's URL only `)` and `\\`, which is MarkdownV2's own rule. + + Two entities of the same kind side by side (`_a__b_`) would read as underline, which Telegram's + parser takes greedily; the second is written as plain text instead, which loses one emphasis and + never a message. + + Nothing here is a markdown renderer and nothing tries to be: tables, footnotes and HTML are text. + """ + + @reserved ~c"_*[]()~`>#+-=|{}.!\\" + @ascii_punct ~c"!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" + @schemes ~w(http https tg mailto) + @rule "──────────" + + @doc "Markdown as MarkdownV2 text that Telegram's parser accepts, whatever the input." + @spec to_markdown_v2(String.t()) :: String.t() + def to_markdown_v2(text) when is_binary(text) do + text + |> String.split("\n") + |> blocks([]) + |> Enum.map_join("\n", &render_block/1) + end + + @doc "A character escaped for MarkdownV2 text, when it needs it." + @spec escape(String.t()) :: String.t() + def escape(text) when is_binary(text) do + for <>, into: "", do: escape_char(c) + end + + @doc """ + The length Telegram counts: UTF-16 code units, so a character outside the Basic Multilingual + Plane (most emoji) counts two. + """ + @spec utf16_length(String.t()) :: non_neg_integer() + def utf16_length(text) when is_binary(text) do + for <>, reduce: 0, do: (n -> n + if(c > 0xFFFF, do: 2, else: 1)) + end + + ## Blocks: fenced code, and lines + + defp blocks([], acc), do: Enum.reverse(acc) + + defp blocks([line | rest], acc) do + case Regex.run(~r/^ {0,3}```([^`]*)$/, line) do + [_, info] -> + {body, rest} = until_fence(rest, []) + blocks(rest, [{:code, language(info), body} | acc]) + + nil -> + blocks(rest, [{:line, line} | acc]) + end + end + + # An unclosed fence runs to the end of the text and is closed here: what follows an opening + # fence is code, and a stream that has not reached the closing fence yet is still code. + defp until_fence([], acc), do: {Enum.reverse(acc), []} + + defp until_fence([line | rest], acc) do + if Regex.match?(~r/^ {0,3}```\s*$/, line), + do: {Enum.reverse(acc), rest}, + else: until_fence(rest, [line | acc]) + end + + defp language(info) do + case info |> String.trim() |> String.split(~r/\s+/, parts: 2) do + [word | _] -> if Regex.match?(~r/^[A-Za-z0-9]{1,32}$/, word), do: word, else: "" + _ -> "" + end + end + + defp render_block({:code, lang, body}) do + case Enum.join(body, "\n") do + "" -> "" + code -> "```" <> lang <> "\n" <> escape_code(code) <> "\n```" + end + end + + defp render_block({:line, line}) do + cond do + match = Regex.run(~r/^ {0,3}[#]{1,6}[ \t]+(.*?)(?:[ \t]+#+)?[ \t]*$/, line) -> + line_entity(Enum.at(match, 1), "*", MapSet.new([:bold])) + + Regex.match?(~r/^ {0,3}[#]{1,6}[ \t]*$/, line) -> + "" + + Regex.match?(~r/^ {0,3}([-*_])(?:[ \t]*\1){2,}[ \t]*$/, line) -> + @rule + + match = Regex.run(~r/^ {0,3}>[ \t]?(.*)$/, line) -> + quote_line(Enum.at(match, 1)) + + match = Regex.run(~r/^([ \t]*)[-*+][ \t]+(.*)$/, line) -> + [_, indent, rest] = match + indent <> "• " <> inline(rest, MapSet.new()) + + match = Regex.run(~r/^([ \t]*)(\d{1,9})([.)])[ \t]+(.*)$/, line) -> + [_, indent, n, delim, rest] = match + indent <> n <> "\\" <> delim <> " " <> inline(rest, MapSet.new()) + + true -> + inline(line, MapSet.new()) + end + end + + defp line_entity(content, delim, ctx) do + case inline(content, ctx) do + "" -> "" + rendered -> delim <> rendered <> delim + end + end + + defp quote_line(rest) do + case inline(rest, MapSet.new()) do + "" -> "" + rendered -> ">" <> rendered + end + end + + ## Inline + + # `ctx` holds the entity kinds already open around this text: one kind is never nested in + # itself, so the markers of an inner one are dropped and its text kept. + defp inline(text, ctx), do: text |> scan(ctx, nil, nil, []) |> IO.iodata_to_binary() + + # `prev` is the source character before this point (for the word-boundary rule on `_`); + # `last` is the kind of entity just emitted, if the last thing emitted was one. + defp scan("", _ctx, _prev, _last, acc), do: Enum.reverse(acc) + + defp scan(<<"\\", c::utf8, rest::binary>>, ctx, _prev, _last, acc) when c in @ascii_punct, + do: scan(rest, ctx, c, nil, [escape_char(c) | acc]) + + defp scan(<<"`", _::binary>> = text, ctx, _prev, _last, acc) do + case Regex.run(~r/\A(`+)(?!`)(.+?)(? + rest = binary_part(text, byte_size(whole), byte_size(text) - byte_size(whole)) + scan(rest, ctx, ?`, :code, ["`" <> escape_code(inner) <> "`" | acc]) + + nil -> + {ticks, rest} = take_run(text, ?`) + scan(rest, ctx, ?`, nil, [String.duplicate("\\`", byte_size(ticks)) | acc]) + end + end + + defp scan(<<"**", rest::binary>>, ctx, _prev, last, acc), + do: pair(rest, "**", :bold, "*", ctx, last, acc) + + defp scan(<<"~~", rest::binary>>, ctx, _prev, last, acc), + do: pair(rest, "~~", :strike, "~", ctx, last, acc) + + defp scan(<<"__", rest::binary>>, ctx, prev, last, acc) do + if word?(prev), + do: scan(rest, ctx, ?_, nil, ["\\_\\_" | acc]), + else: pair(rest, "__", :bold, "*", ctx, last, acc, &closes_at_word_end?/2) + end + + defp scan(<<"*", rest::binary>>, ctx, _prev, last, acc), + do: pair(rest, "*", :italic, "_", ctx, last, acc) + + defp scan(<<"_", rest::binary>>, ctx, prev, last, acc) do + if word?(prev), + do: scan(rest, ctx, ?_, nil, ["\\_" | acc]), + else: pair(rest, "_", :italic, "_", ctx, last, acc, &closes_at_word_end?/2) + end + + defp scan(<<"![", _::binary>> = text, ctx, prev, last, acc) do + case link(binary_part(text, 1, byte_size(text) - 1), ctx) do + {:ok, rendered, rest} -> scan(rest, ctx, ?), :link, [rendered | acc]) + :error -> plain(text, ctx, prev, last, acc) + end + end + + defp scan(<<"[", _::binary>> = text, ctx, prev, last, acc) do + case link(text, ctx) do + {:ok, rendered, rest} -> scan(rest, ctx, ?), :link, [rendered | acc]) + :error -> plain(text, ctx, prev, last, acc) + end + end + + defp scan(text, ctx, prev, last, acc), do: plain(text, ctx, prev, last, acc) + + defp plain(<>, ctx, _prev, _last, acc), + do: scan(rest, ctx, c, nil, [escape_char(c) | acc]) + + # A byte that is not UTF-8 is not text Telegram accepts; it becomes the replacement character + # rather than a refused message. + defp plain(<<_byte, rest::binary>>, ctx, _prev, _last, acc), + do: scan(rest, ctx, nil, nil, ["\uFFFD" | acc]) + + # A delimited entity: the closing delimiter is the first one that leaves a content that is not + # empty, does not start or end with a space, and passes `closes?` (the word-end rule for `_`). + defp pair(rest, delim, kind, out, ctx, last, acc, closes? \\ fn _, _ -> true end) do + case close(rest, delim, closes?) do + {:ok, inner, after_close} -> + cond do + MapSet.member?(ctx, kind) -> + scan(after_close, ctx, last_char(delim), nil, [inline(inner, ctx) | acc]) + + last == kind -> + # Two of a kind side by side: `_a__b_` reads as underline in MarkdownV2. + scan(after_close, ctx, last_char(delim), nil, [inline(inner, ctx) | acc]) + + true -> + rendered = out <> inline(inner, MapSet.put(ctx, kind)) <> out + scan(after_close, ctx, last_char(delim), kind, [rendered | acc]) + end + + :error -> + scan(rest, ctx, last_char(delim), nil, [escape(delim) | acc]) + end + end + + defp close(rest, delim, closes?) do + rest + |> :binary.matches(delim) + |> Enum.find_value(:error, fn {at, len} -> + inner = binary_part(rest, 0, at) + after_close = binary_part(rest, at + len, byte_size(rest) - at - len) + + if inner != "" and not blank_edge?(inner) and not doubled?(delim, inner, after_close) and + closes?.(inner, after_close), + do: {:ok, inner, after_close} + end) + end + + # A one-character delimiter does not close on half of a doubled one: the `*` of `**` belongs + # to a bold, not to the italic being closed. + defp doubled?(<>, inner, after_close), + do: String.ends_with?(inner, <>) or String.starts_with?(after_close, <>) + + defp doubled?(_delim, _inner, _after_close), do: false + + defp blank_edge?(inner), + do: String.match?(inner, ~r/\A\s/u) or String.match?(inner, ~r/\s\z/u) + + defp closes_at_word_end?(_inner, after_close) do + case after_close do + <> -> not word?(c) + "" -> true + end + end + + defp link(text, ctx) do + url_part = ~S"((?:[^\s()\\]|\([^\s()\\]*\))+)" + + case Regex.run(~r/\A\[([^\[\]\n]+)\]\(#{url_part}\)/u, text) do + [whole, label, url] -> + rest = binary_part(text, byte_size(whole), byte_size(text) - byte_size(whole)) + + if allowed_url?(url) and not MapSet.member?(ctx, :link) do + rendered = + "[" <> inline(label, MapSet.put(ctx, :link)) <> "](" <> escape_url(url) <> ")" + + {:ok, rendered, rest} + else + :error + end + + nil -> + :error + end + end + + defp allowed_url?(url) do + case String.split(url, ":", parts: 2) do + [scheme, rest] when rest != "" -> String.downcase(scheme) in @schemes + _ -> false + end + end + + defp take_run(text, char), do: take_run(text, char, "") + + defp take_run(<>, char, run) when c == char, + do: take_run(rest, char, run <> <>) + + defp take_run(rest, _char, run), do: {run, rest} + + defp last_char(delim), do: :binary.last(delim) + + ## Escaping + + defp escape_char(c) when c in @reserved, do: <> + defp escape_char(c), do: <> + + defp escape_code(text), do: String.replace(text, ~r/[`\\]/, "\\\\\\0") + defp escape_url(url), do: String.replace(url, ~r/[)\\]/, "\\\\\\0") + + defp word?(nil), do: false + defp word?(c), do: String.match?(<>, ~r/\A[\p{L}\p{N}]\z/u) +end diff --git a/lib/trinity/gateways/telegram/media.ex b/lib/trinity/gateways/telegram/media.ex new file mode 100644 index 0000000..93e5bb3 --- /dev/null +++ b/lib/trinity/gateways/telegram/media.ex @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Gateways.Telegram.Media do + @moduledoc """ + Images that arrive in a Telegram message (slice 071): a photo, or a file sent as a document whose + type is an image. + + `attachments/1` reads nothing from Telegram. It answers what the message carries as attachments + the router can fetch, each with a `fetch` function, and the router calls it only after the sender + has been admitted and is inside the rate limit (`Trinity.Gateways.Router`). So a stranger's photo + is never downloaded: docs/07 says nothing of an unknown sender's message is read but the code it + might be, and a file on disk is a lot more than a code. + + A fetched image is stored at `/media/.`, named by its digest so the same + image sent twice is one file. It is refused when Telegram says it is larger than the limit + (`config :trinity, :telegram, max_image_bytes:`, 10 MB by default), when more bytes than that + arrive, and when its first bytes are not the image format it claims to be: the bytes are handed + to a model as an image, so they have to be one. + """ + + alias Trinity.Gateways.Telegram + alias Trinity.Gateways.Telegram.Client + + # Sobelow reads `@sobelow_skip` from the source; the compiler would call it unused otherwise. + Module.register_attribute(__MODULE__, :sobelow_skip, persist: true) + + @image_types %{ + "image/jpeg" => "jpg", + "image/png" => "png", + "image/webp" => "webp", + "image/gif" => "gif" + } + + @typedoc "An attachment as the router receives it." + @type attachment :: %{ + kind: :image, + media_type: String.t(), + fetch: (-> {:ok, map()} | {:error, term()}) + } + + @doc "The images a message carries, as attachments to fetch later. Nothing is downloaded here." + @spec attachments(map()) :: [attachment()] + def attachments(%{"photo" => [_ | _] = sizes}) do + max = Telegram.max_image_bytes() + + # Telegram lists a photo's sizes smallest first; the largest that fits is the one worth + # reading. A size with no reported file_size is taken on trust and checked on arrival. + size = + sizes + |> Enum.filter(&(Map.get(&1, "file_size", 0) <= max)) + |> List.last() + + case size do + %{"file_id" => file_id} -> [attachment(file_id, "image/jpeg")] + _ -> [too_large(List.last(sizes))] + end + end + + def attachments(%{"document" => %{"file_id" => file_id, "mime_type" => type} = doc}) + when is_map_key(@image_types, type) do + if Map.get(doc, "file_size", 0) <= Telegram.max_image_bytes(), + do: [attachment(file_id, type)], + else: [too_large(doc)] + end + + def attachments(_message), do: [] + + defp attachment(file_id, media_type), + do: %{kind: :image, media_type: media_type, fetch: fn -> fetch(file_id, media_type) end} + + defp too_large(size) do + bytes = Map.get(size || %{}, "file_size", 0) + %{kind: :image, media_type: "image/jpeg", fetch: fn -> {:error, {:too_large, bytes}} end} + end + + @doc """ + Downloads an image Telegram holds, checks it, and stores it. Answers the image part a session + row carries: its path, media type, digest, size and origin. + """ + @spec fetch(String.t(), String.t()) :: {:ok, map()} | {:error, term()} + def fetch(file_id, media_type) do + max = Telegram.max_image_bytes() + + with {:ok, %{"file_path" => file_path} = file} <- + Client.call("getFile", %{file_id: file_id}), + :ok <- within(Map.get(file, "file_size", 0), max), + {:ok, bytes} <- Client.download(file_path, max), + :ok <- sniff(bytes, media_type) do + store(bytes, media_type) + else + {:ok, _no_path} -> {:error, :no_file_path} + {:error, _} = error -> error + end + end + + defp within(size, max) when is_integer(size) and size > max, do: {:error, {:too_large, size}} + defp within(_size, _max), do: :ok + + # The first bytes of each format this accepts. A file whose bytes are not the format it claims + # is not handed to a model as an image. + defp sniff(<<0xFF, 0xD8, 0xFF, _::binary>>, "image/jpeg"), do: :ok + defp sniff(<<0x89, "PNG", 0x0D, 0x0A, 0x1A, 0x0A, _::binary>>, "image/png"), do: :ok + defp sniff(<<"RIFF", _size::binary-size(4), "WEBP", _::binary>>, "image/webp"), do: :ok + defp sniff(<<"GIF8", _::binary>>, "image/gif"), do: :ok + defp sniff(_bytes, media_type), do: {:error, {:not_an_image, media_type}} + + # sobelow_skip reason: Traversal.FileModule: the directory is `Telegram.state_dir/0` plus a + # constant, and the file name is the SHA-256 of the bytes in hex and an extension from a fixed + # map; nothing the sender chose (a file name, a caption, Telegram's `file_path`) is in the path. + @sobelow_skip ["Traversal.FileModule"] + defp store(bytes, media_type) do + hex = :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) + dir = Path.join(Telegram.state_dir(), "media") + path = Path.join(dir, hex <> "." <> Map.fetch!(@image_types, media_type)) + File.mkdir_p!(dir) + unless File.exists?(path), do: File.write!(path, bytes) + + {:ok, + %{ + "path" => path, + "media_type" => media_type, + "digest" => "sha256:" <> hex, + "bytes" => byte_size(bytes), + "origin" => "gateway:telegram" + }} + end +end diff --git a/lib/trinity/gateways/telegram/offset.ex b/lib/trinity/gateways/telegram/offset.ex new file mode 100644 index 0000000..3292238 --- /dev/null +++ b/lib/trinity/gateways/telegram/offset.ex @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Gateways.Telegram.Offset do + @moduledoc """ + Where the poller is up to (slice 071, AC5): the `update_id` of the next update to process, + persisted so a restarted poller does not process an update twice. + + Telegram confirms an update only when the next `getUpdates` names a higher offset. A poller that + handled update N and died before asking again would be handed N again, and if it began from + nothing it would handle it again: a turn run twice, which for an agent that acts means its effects + run twice. So the offset is written **before** the update is handed to the router, and read back + at start. That order is at most once: a crash between the write and the routing loses that one + message, which its sender can repeat, and never runs it twice (NOTES, decision 3). + + It is a file, `/offset`, written to a temporary name and renamed over the old one, so a + process killed mid-write leaves the old offset or the new one and never part of either. A file + that is missing or does not hold an integer reads as "nothing processed yet". + """ + + # Sobelow reads `@sobelow_skip` from the source; the compiler would call it unused otherwise. + Module.register_attribute(__MODULE__, :sobelow_skip, persist: true) + + @doc "The stored offset, or nil when nothing has been processed." + # sobelow_skip reason: Traversal.FileModule: the directory is `Telegram.state_dir/0`, the data + # directory plus a constant or the operator's own configuration, and the name is a constant. + @sobelow_skip ["Traversal.FileModule"] + @spec load(Path.t()) :: non_neg_integer() | nil + def load(dir) do + with {:ok, text} <- File.read(path(dir)), + {n, ""} when n >= 0 <- text |> String.trim() |> Integer.parse() do + n + else + _ -> nil + end + end + + @doc "Stores the offset: the next update to process is `offset`, and nothing below it." + # sobelow_skip reason: Traversal.FileModule: as `load/1`; nothing Telegram sends reaches the path. + @sobelow_skip ["Traversal.FileModule"] + @spec store(Path.t(), non_neg_integer()) :: :ok + def store(dir, offset) when is_integer(offset) and offset >= 0 do + File.mkdir_p!(dir) + tmp = path(dir) <> ".tmp" + File.write!(tmp, Integer.to_string(offset)) + File.rename!(tmp, path(dir)) + end + + @doc "The file the offset lives in." + @spec path(Path.t()) :: Path.t() + def path(dir), do: Path.join(dir, "offset") +end diff --git a/lib/trinity/gateways/telegram/outbox.ex b/lib/trinity/gateways/telegram/outbox.ex new file mode 100644 index 0000000..679fd7f --- /dev/null +++ b/lib/trinity/gateways/telegram/outbox.ex @@ -0,0 +1,257 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Gateways.Telegram.Outbox do + @moduledoc """ + What the Telegram adapter sends (slice 071): messages, edits, the typing indicator, and the + adapter's status for `/gateways`. + + **Escaping happens here, once.** The router hands an adapter two kinds of text: chunks it got + from `format/2` (a streamed or finished answer) and text it wrote itself (a pairing prompt, a + command's answer). Both are markdown, and both are converted to MarkdownV2 at this one point, so + every text is escaped exactly once whichever road it came by. If Telegram still refuses to parse + one, the same text is sent again without formatting rather than lost. + + **Edits are throttled to one a second per message.** Telegram rate-limits edits, and the router's + flush interval (700 ms) is the console's, not Telegram's. An edit that comes sooner than a second + after the last one is held, replaced by any newer text, and sent when the second is up, so the + final text of a stream is late by at most a second and never dropped. An edit to the text a + message already shows is not sent at all (Telegram answers it with an error). + + **Typing.** `{:typing, true}` shows "typing..." in the chat and renews it every four seconds + (Telegram clears it after five) until something is sent to that chat or a minute passes. + """ + use GenServer + + alias Trinity.Gateways.Telegram + alias Trinity.Gateways.Telegram.{Client, Markdown} + + require Logger + + @edit_interval_ms 1_000 + @typing_every_ms 4_000 + @typing_for_ms 60_000 + @call_timeout_ms 20_000 + + ## API + + @doc "Starts the outbox." + @spec start_link(keyword()) :: GenServer.on_start() + def start_link(opts \\ []), do: GenServer.start_link(__MODULE__, opts, name: __MODULE__) + + @doc "Delivers one outbound instruction to a chat (the adapter's `deliver/2`)." + @spec deliver(String.t(), term()) :: :ok | {:ok, integer()} | {:error, term()} + def deliver(chat, outbound), + do: GenServer.call(__MODULE__, {:deliver, chat, outbound}, @call_timeout_ms) + + @doc "The adapter's status, as the poller last reported it." + @spec status() :: map() + def status, do: GenServer.call(__MODULE__, :status) + + @doc "Records the adapter's status (the poller's report)." + @spec report(map()) :: :ok + def report(status) when is_map(status), do: GenServer.cast(__MODULE__, {:report, status}) + + @doc "The edit interval, in milliseconds." + @spec edit_interval_ms() :: pos_integer() + def edit_interval_ms, do: @edit_interval_ms + + ## The process + + @impl GenServer + def init(_opts) do + status = + if Client.configured?(), + do: %{state: :starting, detail: "starting"}, + else: %{state: :idle, detail: "idle: #{Client.token_env()} is not set"} + + {:ok, %{edits: %{}, typing: %{}, status: status}} + end + + @impl GenServer + def handle_call({:deliver, chat, outbound}, _from, state) do + {reply, state} = handle_outbound(chat, outbound, state) + {:reply, reply, state} + end + + def handle_call(:status, _from, state), do: {:reply, state.status, state} + + @impl GenServer + def handle_cast({:report, status}, state), do: {:noreply, %{state | status: status}} + + @impl GenServer + def handle_info({:flush_edit, key}, state), do: {:noreply, flush_edit(key, state)} + + # `token` names one run of the indicator, so a renewal left in the mailbox by a run that was + # stopped cannot start a second chain beside a newer one. + def handle_info({:typing, chat, token, until}, state) do + case Map.get(state.typing, chat) do + {^token, _timer} -> + if now() < until do + _ = Client.call("sendChatAction", %{chat_id: chat, action: "typing"}) + timer = Process.send_after(self(), {:typing, chat, token, until}, @typing_every_ms) + {:noreply, put_in(state.typing[chat], {token, timer})} + else + {:noreply, stop_typing(state, chat)} + end + + _other -> + {:noreply, state} + end + end + + ## Outbound + + defp handle_outbound(chat, {:message, text}, state), do: send_message(chat, text, nil, state) + + defp handle_outbound(chat, {:message, text, buttons}, state), + do: send_message(chat, text, keyboard(buttons), state) + + defp handle_outbound(chat, {:edit, message_id, text}, state), + do: {:ok, edit(chat, message_id, text, state)} + + defp handle_outbound(chat, {:typing, true}, state) do + state = stop_typing(state, chat) + token = make_ref() + send(self(), {:typing, chat, token, now() + @typing_for_ms}) + {:ok, put_in(state.typing[chat], {token, nil})} + end + + defp handle_outbound(chat, {:typing, false}, state), do: {:ok, stop_typing(state, chat)} + + defp send_message(chat, text, markup, state) do + state = stop_typing(state, chat) + + case String.trim(text) do + "" -> + {{:error, :empty}, state} + + _ -> + # A text longer than one message after escaping (a long command answer: the router's own + # text is chunked before escaping) goes as several; the first is the one a later edit names. + [first | rest] = Telegram.format(text, Telegram.capabilities()) + reply = post_message(chat, first, if(rest == [], do: markup)) + for chunk <- rest, do: post_message(chat, chunk, nil) + + case reply do + {:ok, %{"message_id" => id}} -> + {{:ok, id}, put_in(state.edits[{chat, id}], %{at: now(), text: first, pending: nil})} + + {:error, reason} -> + {{:error, reason}, state} + end + end + end + + defp post_message(chat, text, markup) do + params = + %{chat_id: chat, text: Markdown.to_markdown_v2(text), parse_mode: "MarkdownV2"} + |> put_markup(markup) + + case Client.call("sendMessage", params) do + {:error, {:telegram, 400, description}} = error -> + if parse_error?(description), + do: Client.call("sendMessage", put_markup(%{chat_id: chat, text: text}, markup)), + else: logged(error, "sendMessage") + + {:error, _} = error -> + logged(error, "sendMessage") + + ok -> + ok + end + end + + defp edit(chat, message_id, text, state) do + key = {chat, message_id} + entry = Map.get(state.edits, key, %{at: 0, text: nil, pending: nil}) + + cond do + entry.text == text and entry.pending == nil -> + state + + entry.pending != nil -> + # A flush is already scheduled; it will send the newest text. + put_in(state.edits[key], %{entry | pending: text}) + + now() - entry.at >= @edit_interval_ms -> + put_edit(chat, message_id, text) + put_in(state.edits[key], %{entry | at: now(), text: text}) + + true -> + Process.send_after(self(), {:flush_edit, key}, entry.at + @edit_interval_ms - now()) + put_in(state.edits[key], %{entry | pending: text}) + end + end + + defp flush_edit({chat, message_id} = key, state) do + case Map.get(state.edits, key) do + %{pending: text} = entry when is_binary(text) -> + if text != entry.text, do: put_edit(chat, message_id, text) + put_in(state.edits[key], %{entry | at: now(), text: text, pending: nil}) + + _ -> + state + end + end + + defp put_edit(chat, message_id, text) do + params = %{ + chat_id: chat, + message_id: message_id, + text: Markdown.to_markdown_v2(text), + parse_mode: "MarkdownV2" + } + + case Client.call("editMessageText", params) do + {:error, {:telegram, 400, description}} = error -> + cond do + description =~ "message is not modified" -> + :ok + + parse_error?(description) -> + Client.call("editMessageText", %{chat_id: chat, message_id: message_id, text: text}) + + true -> + logged(error, "editMessageText") + end + + {:error, _} = error -> + logged(error, "editMessageText") + + _ok -> + :ok + end + end + + defp keyboard(buttons) do + %{ + inline_keyboard: [ + for({label, command} <- buttons, do: %{text: label, callback_data: command}) + ] + } + end + + defp put_markup(params, nil), do: params + defp put_markup(params, markup), do: Map.put(params, :reply_markup, markup) + + defp parse_error?(description), do: description =~ "can't parse entities" + + defp stop_typing(state, chat) do + case Map.pop(state.typing, chat) do + {nil, _} -> + state + + {{_token, timer}, typing} -> + if is_reference(timer), do: Process.cancel_timer(timer) + %{state | typing: typing} + end + end + + # The reason is the client's, which never carries the token or a URL; the text is not logged. + defp logged({:error, reason} = error, method) do + Logger.warning("telegram: #{method} failed: #{inspect(reason)}") + error + end + + defp now, do: System.monotonic_time(:millisecond) +end diff --git a/lib/trinity/gateways/telegram/poller.ex b/lib/trinity/gateways/telegram/poller.ex new file mode 100644 index 0000000..18d4a6a --- /dev/null +++ b/lib/trinity/gateways/telegram/poller.ex @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Gateways.Telegram.Poller do + @moduledoc """ + Long-polling `getUpdates` (slice 071), the default way a bot hears from Telegram. + + One request at a time, held open by Telegram for up to `poll_timeout_s` seconds (50 by + default) until there is something to say. Each update is processed in `update_id` order, and + **its offset is persisted before it is routed** (`Trinity.Gateways.Telegram.Offset`), so a + poller killed at any point resumes after the last update it began and never processes one twice + (AC5). An update below the stored offset is skipped even if the server hands it out again. + + A failed request (Telegram down, a wrong token, a webhook set on the bot so that `getUpdates` is + refused) is reported to `/gateways` through the outbox and retried with a backoff that doubles + to a minute; nothing about it is fatal, and nothing logged carries the token. + + Without `TELEGRAM_BOT_TOKEN` the poller does not start (`:ignore`) and the status says why, so a + desktop with Telegram configured and no token boots and says so rather than crash-looping. + """ + use GenServer + + alias Trinity.Gateways.Telegram + alias Trinity.Gateways.Telegram.{Client, Offset, Outbox, Updates} + + require Logger + + @max_backoff_ms 60_000 + + @doc "Starts the poller." + @spec start_link(keyword()) :: GenServer.on_start() + def start_link(opts \\ []), do: GenServer.start_link(__MODULE__, opts, name: __MODULE__) + + @impl GenServer + def init(_opts) do + if Client.configured?() do + send(self(), :poll) + {:ok, %{offset: Offset.load(Telegram.state_dir()), me: nil, backoff: 0}} + else + Logger.warning("telegram: #{Client.token_env()} is not set; the Telegram gateway is idle") + :ignore + end + end + + @impl GenServer + def handle_info(:poll, %{me: nil} = state) do + case Client.call("getMe") do + {:ok, %{"id" => _, "username" => username} = me} -> + Outbox.report(%{state: :running, detail: "polling as @#{username}"}) + handle_info(:poll, %{state | me: me, backoff: 0}) + + {:ok, _other} -> + retry(state, :unexpected_get_me) + + {:error, reason} -> + retry(state, reason) + end + end + + def handle_info(:poll, state) do + params = + %{timeout: Telegram.poll_timeout_s(), allowed_updates: ["message", "callback_query"]} + |> then(fn p -> if state.offset, do: Map.put(p, :offset, state.offset), else: p end) + + receive_timeout = (Telegram.poll_timeout_s() + 10) * 1_000 + + case Client.call("getUpdates", params, receive_timeout: receive_timeout) do + {:ok, updates} when is_list(updates) -> + if state.backoff > 0, + do: Outbox.report(%{state: :running, detail: "polling as @#{state.me["username"]}"}) + + state = Enum.reduce(Enum.sort_by(updates, & &1["update_id"]), state, &process/2) + send(self(), :poll) + {:noreply, %{state | backoff: 0}} + + {:ok, _other} -> + retry(state, :unexpected_updates) + + {:error, reason} -> + retry(state, reason) + end + end + + def handle_info(_other, state), do: {:noreply, state} + + # The offset is stored first and the update routed second: at most once (NOTES, decision 3). + defp process(%{"update_id" => id} = update, state) when is_integer(id) do + if state.offset && id < state.offset do + state + else + :ok = Offset.store(Telegram.state_dir(), id + 1) + route(update, state.me) + %{state | offset: id + 1} + end + end + + defp process(_malformed, state), do: state + + # One update that fails to route costs that update and not the poller: its offset is already + # stored, so a crash here would only restart the loop to skip it. The reason is logged; the + # message text is not. + defp route(update, me) do + Updates.route(update, me) + rescue + error -> Logger.error("telegram: update not routed: #{inspect(error.__struct__)}") + catch + :exit, reason -> + Logger.error("telegram: update not routed: exit #{inspect(elem_reason(reason))}") + end + + defp elem_reason({reason, _call}), do: reason + defp elem_reason(reason), do: reason + + defp retry(state, reason) do + backoff = min(max(state.backoff * 2, 1_000), @max_backoff_ms) + Logger.warning("telegram: polling failed (#{inspect(reason)}); retrying in #{backoff} ms") + Outbox.report(%{state: :error, detail: "error: #{describe(reason)}; retrying"}) + Process.send_after(self(), :poll, backoff) + {:noreply, %{state | backoff: backoff}} + end + + defp describe({:telegram, 401, _}), do: "Telegram refused the token (401)" + + defp describe({:telegram, 409, _}), + do: + "a webhook is set on this bot, so getUpdates is refused (409); remove it with deleteWebhook" + + defp describe({:telegram, code, description}), do: "Telegram #{code}: #{description}" + defp describe({:transport, reason}), do: "cannot reach Telegram (#{reason})" + defp describe(other), do: inspect(other) +end diff --git a/lib/trinity/gateways/telegram/updates.ex b/lib/trinity/gateways/telegram/updates.ex new file mode 100644 index 0000000..c884596 --- /dev/null +++ b/lib/trinity/gateways/telegram/updates.ex @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Gateways.Telegram.Updates do + @moduledoc """ + One Telegram update, handed to the router (slice 071). This is the adapter's whole inbound side, + and it holds 070's rule: it reads what Telegram sent, decides whether the bot was spoken to, and + calls `Trinity.Gateways.Router.inbound/5`. It never touches a session, never calls a model and + never decides an approval. + + **Who and where.** The conversation is the chat (`chat.id`) and the identity is the sender + (`from.id`), so in a group each person pairs for themselves and one session serves the chat. + + **When the bot was spoken to.** Every message in a private chat. In a group, a message that + mentions the bot (`@name`), replies to one of its messages, or is a command addressed to it + (`/help@name`); a group's other traffic is not the bot's business and is not read. Messages from + bots and from channels are ignored. + + **`/start`.** Telegram sends it when a person first opens the bot. It is not one of the router's + commands, so it is translated: `/start` alone becomes `/help` (an unpaired sender is shown the + pairing prompt whatever they say, and a paired one sees what the bot answers); `/start ` + becomes the code, so a `t.me/?start=` link pairs in one tap. + + **Buttons.** A press of an approval button is routed as the command its button carries + (`/approve ` or `/deny `), from the person who pressed it, exactly as if they had typed it: + the identity check, the rate limit, the channel cap and the gate all apply. Callback data that is + not one of those two commands is answered and ignored. A forged press can therefore say nothing + its sender could not have typed. + + **Images.** A photo, or an image sent as a file, becomes an attachment the router fetches after + admission (`Trinity.Gateways.Telegram.Media`). Its caption is the text; with no caption the text is + `(image)`, because a user message is never empty. + """ + + alias Trinity.Gateways.Router + alias Trinity.Gateways.Telegram + alias Trinity.Gateways.Telegram.{Client, Media} + + @decision ~r/\A\/(approve|deny) [0-9A-Za-z-]{1,36}\z/ + @code ~r/\A[A-Z2-9]{6}\z/ + + @doc "Routes one update; answers what the router said, or `:ignored`." + @spec route(map(), map()) :: term() + def route(%{"message" => message}, me), do: message(message, me) + def route(%{"callback_query" => query}, me), do: callback(query, me) + def route(_update, _me), do: :ignored + + defp message(%{"from" => %{"is_bot" => true}}, _me), do: :ignored + defp message(%{"chat" => %{"type" => "channel"}}, _me), do: :ignored + + defp message(%{"from" => from, "chat" => %{"id" => chat_id} = chat} = message, me) do + text = Map.get(message, "text") || Map.get(message, "caption") || "" + attachments = Media.attachments(message) + + if addressed?(chat, message, text, me) do + text = text |> strip_mention(me) |> start_command() + inbound(chat_id, from, text, attachments) + else + :ignored + end + end + + defp message(_message, _me), do: :ignored + + defp inbound(_chat_id, _from, "", []), do: :ignored + + defp inbound(chat_id, from, text, attachments) do + text = if String.trim(text) == "", do: "(image)", else: text + conversation = to_string(chat_id) + + result = + Router.inbound(Telegram, conversation, to_string(from["id"]), text, + display_name: display_name(from), + attachments: attachments + ) + + if result == {:ok, :placed}, do: Telegram.deliver(conversation, {:typing, true}) + result + end + + defp callback(%{"id" => id, "from" => from} = query, _me) do + _ = Client.call("answerCallbackQuery", %{callback_query_id: id}) + + with %{"message" => %{"chat" => %{"id" => chat_id}, "message_id" => message_id}} <- query, + data when is_binary(data) <- Map.get(query, "data"), + true <- Regex.match?(@decision, data) do + result = + Router.inbound(Telegram, to_string(chat_id), to_string(from["id"]), data, + display_name: display_name(from) + ) + + # The request is answered (or gone) once the command ran, so the buttons come off the + # message rather than inviting a second press. + if result == {:ok, :command} do + _ = + Client.call("editMessageReplyMarkup", %{ + chat_id: chat_id, + message_id: message_id, + reply_markup: %{inline_keyboard: []} + }) + end + + result + else + _ -> :ignored + end + end + + defp callback(_query, _me), do: :ignored + + ## Was the bot spoken to? + + defp addressed?(%{"type" => "private"}, _message, _text, _me), do: true + + defp addressed?(%{"type" => type}, message, text, me) when type in ["group", "supergroup"] do + username = Map.get(me, "username", "") + + reply_to_bot?(message, me) or + (username != "" and + String.contains?(String.downcase(text), "@" <> String.downcase(username))) + end + + defp addressed?(_chat, _message, _text, _me), do: false + + defp reply_to_bot?(%{"reply_to_message" => %{"from" => %{"id" => id}}}, %{"id" => id}), do: true + defp reply_to_bot?(_message, _me), do: false + + # `/help@bot args` is `/help args`; `@bot` elsewhere in the text is the address, not the content. + defp strip_mention(text, %{"username" => username}) when is_binary(username) do + mention = ~r/@#{Regex.escape(username)}\b/i + + case Regex.run(~r/\A(\/[A-Za-z0-9_]+)@#{Regex.escape(username)}\b(.*)\z/is, text) do + [_, command, rest] -> command <> rest + nil -> text |> String.replace(mention, "") |> String.trim() + end + end + + defp strip_mention(text, _me), do: text + + defp start_command(text) do + case String.split(String.trim(text), ~r/\s+/, parts: 2) do + ["/start"] -> "/help" + ["/start", payload] -> if Regex.match?(@code, payload), do: payload, else: "/help" + _ -> text + end + end + + defp display_name(from) do + [Map.get(from, "first_name"), Map.get(from, "last_name")] + |> Enum.reject(&(&1 in [nil, ""])) + |> case do + [] -> Map.get(from, "username") + names -> Enum.join(names, " ") + end + end +end diff --git a/lib/trinity/llm.ex b/lib/trinity/llm.ex index e4ce445..fc5a355 100644 --- a/lib/trinity/llm.ex +++ b/lib/trinity/llm.ex @@ -22,7 +22,7 @@ defmodule Trinity.LLM do def stream(%Request{} = request, opts \\ [], emit) when is_function(emit, 1) do with {:ok, entry, module} <- resolve(request) do call(entry, "chat", opts, fn -> - module.stream(request, provider_opts(entry, opts), emit) + module.stream(fit(request, entry), provider_opts(entry, opts), emit) end) end end @@ -49,7 +49,9 @@ defmodule Trinity.LLM do @spec generate(Request.t(), opts()) :: {:ok, Trinity.LLM.Provider.result()} | {:error, term()} def generate(%Request{} = request, opts \\ []) do with {:ok, entry, module} <- resolve(request) do - call(entry, "chat", opts, fn -> module.generate(request, provider_opts(entry, opts)) end) + call(entry, "chat", opts, fn -> + module.generate(fit(request, entry), provider_opts(entry, opts)) + end) end end @@ -59,7 +61,7 @@ defmodule Trinity.LLM do with {:ok, entry, module} <- resolve(request), {:ok, object, _usage} <- call(entry, "object", opts, fn -> - module.generate_object(request, schema, provider_opts(entry, opts)) + module.generate_object(fit(request, entry), schema, provider_opts(entry, opts)) end) do {:ok, object} end @@ -90,6 +92,9 @@ defmodule Trinity.LLM do with {:ok, entry} <- Registry.lookup(model_id), do: {:ok, entry.caps} end + # Slice 071: images reach a provider only for a model whose registry entry declares `:vision`. + defp fit(request, entry), do: Request.fit_images(request, :vision in Map.get(entry, :caps, [])) + defp resolve(%Request{model: model}) do with {:ok, entry} <- Registry.lookup(model), {:ok, module} <- Registry.provider_module(entry) do diff --git a/lib/trinity/llm/providers/req_llm.ex b/lib/trinity/llm/providers/req_llm.ex index ad5f771..77d70ef 100644 --- a/lib/trinity/llm/providers/req_llm.ex +++ b/lib/trinity/llm/providers/req_llm.ex @@ -16,6 +16,9 @@ defmodule Trinity.LLM.Providers.ReqLLM do """ @behaviour Trinity.LLM.Provider + # Sobelow reads `@sobelow_skip` from the source; the compiler would call it unused otherwise. + Module.register_attribute(__MODULE__, :sobelow_skip, persist: true) + alias Trinity.Config alias Trinity.LLM.{Error, Request} alias Trinity.LLM.Providers.ReqLLM.Mapping @@ -204,6 +207,24 @@ defmodule Trinity.LLM.Providers.ReqLLM do end defp message(%{role: "system", content: c}), do: ReqLLM.Context.system(c) + # Slice 071: a user message with images is a list of content parts, the text first. An image + # file that cannot be read any more (removed from the data directory) is said in words rather + # than failing the turn. + defp message(%{role: "user", content: c, images: [_ | _] = images}) do + {parts, missing} = + Enum.reduce(images, {[], 0}, fn %{path: path, media_type: type}, {parts, missing} -> + case read_image(path) do + {:ok, bytes} -> {parts ++ [ReqLLM.Message.ContentPart.image(bytes, type)], missing} + {:error, _} -> {parts, missing + 1} + end + end) + + text = + if missing > 0, do: c <> "\n\n[#{missing} attached image(s) could not be read]", else: c + + ReqLLM.Context.user([ReqLLM.Message.ContentPart.text(text) | parts]) + end + defp message(%{role: "user", content: c}), do: ReqLLM.Context.user(c) defp message(%{role: "tool", content: c} = m), @@ -223,6 +244,12 @@ defmodule Trinity.LLM.Providers.ReqLLM do end end + # sobelow_skip reason: Traversal.FileModule: the path is one the gateway stored itself, under the + # data directory and named by the image's SHA-256 digest (slice 071, `Telegram.Media`), read back + # from the session's own row; no request names it. + @sobelow_skip ["Traversal.FileModule"] + defp read_image(path), do: File.read(path) + defp maybe_put(opts, _key, nil), do: opts defp maybe_put(opts, key, value), do: Keyword.put(opts, key, value) end diff --git a/lib/trinity/llm/request.ex b/lib/trinity/llm/request.ex index 9591976..5e93c61 100644 --- a/lib/trinity/llm/request.ex +++ b/lib/trinity/llm/request.ex @@ -5,8 +5,8 @@ defmodule Trinity.LLM.Request do What a caller asks a provider for. Slice 011. `messages` are maps with `role` and `content` (and `tool_call_id` for tool results, `tool_calls` - for assistant turns that made them), in the shape the `messages` table stores; the adapter maps - them to the provider's format. `tools` are maps with `name`, `description` and a JSON Schema + for assistant turns that made them, `images` for a user message that came with images, slice 071), + in the shape the `messages` table stores; the adapter maps them to the provider's format. `tools` are maps with `name`, `description` and a JSON Schema under `parameters`. `params` carries `max_tokens`, `temperature` and provider hints such as `cache: true`. `model` is a registry id (`"provider:model"`); nil means the registry default. """ @@ -17,8 +17,11 @@ defmodule Trinity.LLM.Request do required(:role) => String.t(), required(:content) => String.t(), optional(:tool_call_id) => String.t(), - optional(:tool_calls) => [map()] + optional(:tool_calls) => [map()], + optional(:images) => [image()] } + @typedoc "An image a user message carries: a file on this machine and its media type (slice 071)." + @type image :: %{required(:path) => String.t(), required(:media_type) => String.t()} @type tool :: %{ required(:name) => String.t(), required(:description) => String.t(), @@ -55,10 +58,36 @@ defmodule Trinity.LLM.Request do end end + @doc """ + The request a model with or without vision can take (slice 071). With `vision?` it is unchanged. + Without it, every message's images are removed and replaced by a sentence in its text saying an + image was attached and this model cannot see it, so the turn runs and the model can say so rather + than answer as if nothing had been sent. The decision is the registry's (`caps: [:vision]`), never + the caller's. + """ + @spec fit_images(t(), boolean()) :: t() + def fit_images(%__MODULE__{} = request, true), do: request + + def fit_images(%__MODULE__{messages: messages} = request, false) do + %{request | messages: Enum.map(messages, &without_images/1)} + end + + defp without_images(%{images: [_ | _] = images} = message) do + note = + "[#{length(images)} image(s) attached; the model in use does not accept images, " <> + "so they were not sent to it]" + + message |> Map.delete(:images) |> Map.update!(:content, &(&1 <> "\n\n" <> note)) + end + + defp without_images(message), do: Map.delete(message, :images) + defp check_messages(messages) when is_list(messages) do Enum.reduce_while(messages, :ok, fn - %{role: role, content: content}, :ok when role in @roles and is_binary(content) -> - {:cont, :ok} + %{role: role, content: content} = message, :ok when role in @roles and is_binary(content) -> + if images_ok?(message), + do: {:cont, :ok}, + else: {:halt, {:error, {:invalid_request, {:images, message}}}} other, :ok -> {:halt, {:error, {:invalid_request, {:message, other}}}} @@ -67,6 +96,15 @@ defmodule Trinity.LLM.Request do defp check_messages(other), do: {:error, {:invalid_request, {:messages, other}}} + # Images belong to a user message and are a list of paths with media types; anything else is a + # malformed request and refused by name rather than half-sent. + defp images_ok?(%{images: images, role: "user"}) when is_list(images), + do: + Enum.all?(images, &match?(%{path: p, media_type: t} when is_binary(p) and is_binary(t), &1)) + + defp images_ok?(%{images: _}), do: false + defp images_ok?(_message), do: true + defp check_tools(tools) when is_list(tools) do Enum.reduce_while(tools, :ok, fn %{name: name, parameters: %{} = _schema}, :ok when is_binary(name) -> {:cont, :ok} diff --git a/lib/trinity/sessions.ex b/lib/trinity/sessions.ex index 3a5321e..407e991 100644 --- a/lib/trinity/sessions.ex +++ b/lib/trinity/sessions.ex @@ -212,10 +212,19 @@ defmodule Trinity.Sessions do end end - @doc "Persists the user's message and starts a turn; refuses while a turn is in flight." - @spec send_user_message(session_id(), String.t()) :: {:ok, Message.t()} | {:error, term()} - def send_user_message(session_id, content) do - with {:ok, pid} <- ensure_started(session_id), do: Session.send_user_message(pid, content) + @doc """ + Persists the user's message and starts a turn; refuses while a turn is in flight. + + `images:` (slice 071) is a list of image parts that came with the message, each a map with + `"path"`, `"media_type"`, `"digest"`, `"bytes"` and `"origin"`. They are stored in the row's + `parts` and reach the model as images when its registry entry declares `:vision`, and as a + sentence saying an image was attached when it does not (`Trinity.LLM.Request.fit_images/2`). + """ + @spec send_user_message(session_id(), String.t(), keyword()) :: + {:ok, Message.t()} | {:error, term()} + def send_user_message(session_id, content, opts \\ []) do + with {:ok, pid} <- ensure_started(session_id), + do: Session.send_user_message(pid, content, opts) end @doc "Stops the turn in flight, persisting what arrived as interrupted." diff --git a/lib/trinity/sessions/prompt.ex b/lib/trinity/sessions/prompt.ex index 644714f..4dc22af 100644 --- a/lib/trinity/sessions/prompt.ex +++ b/lib/trinity/sessions/prompt.ex @@ -237,6 +237,20 @@ defmodule Trinity.Sessions.Prompt do end end + # Slice 071: a user row that came with images carries them to the request as paths and media + # types; whether the model sees them is the LLM layer's call, from the model's capabilities. + defp message(%Message{role: "user", parts: %{"images" => [_ | _] = images}, content: content}) do + %{ + role: "user", + content: content, + images: + for( + %{"path" => path, "media_type" => type} <- images, + do: %{path: path, media_type: type} + ) + } + end + defp message(%Message{role: role, content: content}), do: %{role: role, content: content} defp tool_content(%Message{parts: parts, content: content}) do diff --git a/lib/trinity/sessions/session.ex b/lib/trinity/sessions/session.ex index c9e0941..2eb0499 100644 --- a/lib/trinity/sessions/session.ex +++ b/lib/trinity/sessions/session.ex @@ -40,10 +40,21 @@ defmodule Trinity.Sessions.Session do @spec via(String.t()) :: {:via, Registry, {Trinity.Registry, String.t()}} def via(session_id), do: {:via, Registry, {Trinity.Registry, session_id}} - @doc "Persists the user's message, broadcasts it, and starts a turn. Busy sessions refuse." - @spec send_user_message(pid() | String.t(), String.t()) :: + @doc """ + Persists the user's message, broadcasts it, and starts a turn. Busy sessions refuse. `images:` + (slice 071) are stored in the row's `parts` as `"images"`; see `Trinity.Sessions.send_user_message/3`. + """ + @spec send_user_message(pid() | String.t(), String.t(), keyword()) :: {:ok, Store.message()} | {:error, term()} - def send_user_message(ref, content), do: :gen_statem.call(target(ref), {:user_message, content}) + def send_user_message(ref, content, opts \\ []) do + parts = + case Keyword.get(opts, :images, []) do + [] -> %{} + images when is_list(images) -> %{"images" => images} + end + + :gen_statem.call(target(ref), {:user_message, content, parts}) + end @doc "Stops the turn in flight, persisting what arrived as interrupted." @spec cancel_turn(pid() | String.t()) :: :ok | {:error, :idle} @@ -160,8 +171,8 @@ defmodule Trinity.Sessions.Session do {:keep_state_and_data, [{:reply, from, view}]} end - def handle_event({:call, from}, {:user_message, content}, :idle, %State{id: id} = data) do - case Trinity.Sessions.append_message(id, %{role: "user", content: content}) do + def handle_event({:call, from}, {:user_message, content, parts}, :idle, %State{id: id} = data) do + case Trinity.Sessions.append_message(id, %{role: "user", content: content, parts: parts}) do {:ok, message} -> Events.broadcast(id, {:user_message, message}) data = %{data | turn: State.new_turn()} @@ -180,7 +191,7 @@ defmodule Trinity.Sessions.Session do end end - def handle_event({:call, from}, {:user_message, _}, state, _data) do + def handle_event({:call, from}, {:user_message, _, _}, state, _data) do {:keep_state_and_data, [{:reply, from, {:error, {:busy, state}}}]} end diff --git a/lib/trinity/telemetry/redaction.ex b/lib/trinity/telemetry/redaction.ex index db468a6..f3d88b5 100644 --- a/lib/trinity/telemetry/redaction.ex +++ b/lib/trinity/telemetry/redaction.ex @@ -54,6 +54,10 @@ defmodule Trinity.Telemetry.Redaction do {~r/(?i)\b(bearer|basic|token)\s+[A-Za-z0-9._~+\/=-]{12,}/, "\\1 [REDACTED]"}, {~r/\b(?:sk|pk|rk|api|key)[-_][A-Za-z0-9._-]{16,}/i, "[REDACTED]"}, {~r/\bey[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/, "[REDACTED-JWT]"}, + # Slice 071: a Telegram bot token, `:`, which the Bot API carries in the + # URL path (`/bot/`) where no key name precedes it. Not anchored on `\b`, since + # in that path the token follows the letters "bot" with no boundary between them. + {~r/(? 0.5", lock: "req", note: "HTTP client."}, + %{ + name: "req", + pin: "~> 0.5", + lock: "req", + note: + "HTTP client. Since Slice 071 also the Telegram Bot API client, in place of a Telegram library." + }, %{ name: "req_llm", pin: "~> 1.22", @@ -322,10 +328,17 @@ defmodule Trinity.Versions do }, %{ name: "telegex", - pin: "**not pinned**", + pin: "not used (measured at Slice 071, 2026-10-08)", + lock: nil, + note: + "Telegram. Measured at Slice 071: the latest release on hex is `1.9.0-rc.0` (2024-09-18) and the last stable `1.8.0` (2024-05-28); this file's own rule forbids pinning an `-rc`. Not used: `Trinity.Gateways.Telegram` speaks the Bot API with `req` (slice 071 NOTES, \"The library, measured\"). The row stays so the decision is visible where a reader would look for the package." + }, + %{ + name: "ex_gram", + pin: "not used (measured at Slice 071, 2026-10-08)", lock: nil, note: - "Telegram. ⚠️ The latest release on hex is a release candidate, roughly two years old, and this file's own rule forbids pinning an `-rc`. Alternative: ex_gram. Slice 071 decides with the measurement." + "Telegram. Measured at Slice 071 on `0.71.0` (2026-10-01): its poller keeps the `getUpdates` offset in process state and starts at `-1`, so a restarted poller reprocesses or skips unconfirmed updates (Slice 071's AC5 is that property); its file download host is a module attribute that cannot be pointed at a test server; Beerware licence; 18,054 lines for what is here a handful of JSON calls. Not used: `Trinity.Gateways.Telegram` speaks the Bot API with `req` and owns its poller." }, %{ name: "phoenix_streamdown", diff --git a/lib/trinity_web/live/gateways_live.ex b/lib/trinity_web/live/gateways_live.ex index 2148f8d..fd4bf22 100644 --- a/lib/trinity_web/live/gateways_live.ex +++ b/lib/trinity_web/live/gateways_live.ex @@ -40,11 +40,22 @@ defmodule TrinityWeb.GatewaysLive do name: Adapter.name(adapter), running?: is_pid(Process.whereis(adapter)), ceiling: Cap.ceiling(adapter), - capabilities: adapter.capabilities() + capabilities: adapter.capabilities(), + status: status(adapter) } end end + # Slice 071: an adapter that reports more than whether it runs (`Adapter.status/0`, optional) + # says what it is doing: which account it is connected as, idle and why, or the last error. + defp status(adapter) do + if Process.whereis(adapter) && function_exported?(adapter, :status, 0) do + adapter.status() + end + catch + :exit, _ -> nil + end + @impl true def handle_info(:refresh, socket) do Process.send_after(self(), :refresh, @refresh_ms) @@ -106,6 +117,17 @@ defmodule TrinityWeb.GatewaysLive do {gettext("max")} {adapter.capabilities.max_length} + + {adapter.status.detail} +

diff --git a/scripts/dev_gateways.sh b/scripts/dev_gateways.sh index d6e81cd..df4f7ad 100755 --- a/scripts/dev_gateways.sh +++ b/scripts/dev_gateways.sh @@ -31,8 +31,9 @@ end Logger.configure(level: :warning) alias Trinity.Gateways.{Console, Identities, Router} -{:ok, _} = Console.start_link([]) -{:ok, _} = Router.start_link([]) +# Slice 071: the application starts the configured gateways itself (Trinity.Gateways.Supervisor), +# so the router and the console are already running here. +true = is_pid(Process.whereis(Console)) and is_pid(Process.whereis(Router)) # The fake provider answers, so the transcript is a real turn without a key. Trinity.LLM.Providers.Fake.scripts([ diff --git a/test/secrets_scan_test.exs b/test/secrets_scan_test.exs index e1341f7..073998a 100644 --- a/test/secrets_scan_test.exs +++ b/test/secrets_scan_test.exs @@ -18,6 +18,14 @@ defmodule SecretsScanTest do assert [{"generic long secret assignment", 2}] = Scan.findings(text) end + # Slice 071. A Telegram bot token is the bot's numeric id, a colon, and 35 characters that + # begin "AA". Composed, as above. The suite's own fake token is not this shape on purpose. + test "RED: a Telegram bot token is found; the suite's fake token is not one" do + token = "123456789:" <> "AA" <> String.duplicate("b", 33) + assert [{"Telegram bot token", 1}] = Scan.findings("TELEGRAM_BOT_TOKEN=" <> token) + assert Scan.findings("071000001:TEST_fake_token_for_the_suite_only") == [] + end + test "GREEN: ordinary source is clean" do assert Scan.findings("def hello, do: :world\n# AKIA is mentioned but not a key\n") == [] end diff --git a/test/support/telegram/fake_bot_api.ex b/test/support/telegram/fake_bot_api.ex new file mode 100644 index 0000000..672b40e --- /dev/null +++ b/test/support/telegram/fake_bot_api.ex @@ -0,0 +1,413 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.FakeTelegram.BotApi do + @moduledoc """ + A fake Telegram Bot API (slice 071), behind Bandit on a loopback port, so the adapter's whole + path (Req, the URL with the token in it, JSON both ways, the file host) runs in the suite with no + network. + + It keeps the parts of Telegram's behaviour the adapter depends on, and is strict where Telegram + is: a wrong token is `401`; `getUpdates` confirms every update below the `offset` it is given and + hands out the rest again until they are confirmed (which is what makes a duplicate possible, and + so what AC5 is about), and holds the request open until there is an update or the timeout is up; + a MarkdownV2 text that `Trinity.FakeTelegram.MarkdownV2` refuses is `400 Bad Request: can't parse + entities`; an edit to the text a message already shows is `400 ... message is not modified`; a + text longer than 4096 UTF-16 units is `400 ... message is too long`. + + A test drives it with `push/2` (an update), `put_file/3` (a file `getFile` can name), `hold/2` (a + request that matching it waits for `release/1`, so a test can act while the adapter is mid-call) + and reads it with `requests/1`. + """ + @behaviour Plug + + import Plug.Conn + + alias Trinity.FakeTelegram.MarkdownV2 + + @token "071000001:TEST_fake_token_for_the_suite_only" + + @doc "The token the fake accepts. It is not shaped like a real one, so no scanner mistakes it." + def token, do: @token + + @doc """ + Starts the fake and points the adapter at it: the base URL, a state directory of the test's own, + a one-second long poll, and the token in the environment. Everything is restored on exit. + """ + def start!(opts \\ []) do + me = %{"id" => 7_100_000, "is_bot" => true, "username" => "trinity_test_bot"} + + # Not linked to the test: a long poll in flight when the test ends would otherwise find the + # agent gone. It is stopped on exit, after the server. + {:ok, agent} = + Agent.start(fn -> + %{ + updates: [], + next_update: 1, + next_message: 100, + messages: %{}, + requests: [], + files: %{}, + me: me, + hold: nil, + watcher: self(), + fail: %{} + } + end) + + {:ok, server} = + Bandit.start_link( + plug: {__MODULE__, agent}, + ip: {127, 0, 0, 1}, + port: 0, + startup_log: false + ) + + {:ok, {_ip, port}} = ThousandIsland.listener_info(server) + base_url = "http://127.0.0.1:#{port}" + + state_dir = + Keyword.get_lazy(opts, :state_dir, fn -> + Path.join(System.tmp_dir!(), "telegram-fake-#{System.unique_integer([:positive])}") + end) + + previous_env = Application.get_env(:trinity, :telegram) + previous_token = System.get_env("TELEGRAM_BOT_TOKEN") + + Application.put_env(:trinity, :telegram, + base_url: base_url, + state_dir: state_dir, + poll_timeout_s: 1, + max_image_bytes: Keyword.get(opts, :max_image_bytes, 1_000_000) + ) + + if Keyword.get(opts, :token, true), do: System.put_env("TELEGRAM_BOT_TOKEN", @token) + + # `on_exit: false` for a script outside ExUnit (the proof run), which stops nothing. + if Keyword.get(opts, :on_exit, true), + do: register_cleanup(server, agent, state_dir, previous_env, previous_token) + + %{agent: agent, base_url: base_url, state_dir: state_dir, me: me} + end + + defp register_cleanup(server, agent, state_dir, previous_env, previous_token) do + ExUnit.Callbacks.on_exit(fn -> + restore_env(previous_env) + restore_token(previous_token) + File.rm_rf(state_dir) + + for pid <- [server, agent] do + try do + GenServer.stop(pid) + catch + :exit, _ -> :ok + end + end + end) + end + + defp restore_env(nil), do: Application.delete_env(:trinity, :telegram) + defp restore_env(env), do: Application.put_env(:trinity, :telegram, env) + defp restore_token(nil), do: System.delete_env("TELEGRAM_BOT_TOKEN") + defp restore_token(token), do: System.put_env("TELEGRAM_BOT_TOKEN", token) + + ## Driving it + + @doc "Queues an update; it is given the next `update_id`, which is returned." + def push(%{agent: agent}, update) do + Agent.get_and_update(agent, fn state -> + id = state.next_update + + {id, + %{ + state + | updates: state.updates ++ [Map.put(update, "update_id", id)], + next_update: id + 1 + }} + end) + end + + @doc "A private-chat text message from `user_id`, as an update body." + def text_message(user_id, text, extra \\ %{}) do + %{ + "message" => + Map.merge( + %{ + "message_id" => System.unique_integer([:positive]), + "from" => %{"id" => user_id, "is_bot" => false, "first_name" => "Ayla"}, + "chat" => %{"id" => user_id, "type" => "private"}, + "date" => 0, + "text" => text + }, + extra + ) + } + end + + @doc "A file `getFile` can name, with the bytes the file host serves for it." + def put_file(%{agent: agent}, file_id, bytes) do + Agent.update(agent, fn state -> + put_in(state.files[file_id], %{path: "photos/#{file_id}.jpg", bytes: bytes}) + end) + end + + @doc "Makes the next `n` calls of `method` answer with this error body (code, description)." + def fail(%{agent: agent}, method, n, code, description) do + Agent.update(agent, &put_in(&1.fail[method], {n, code, description})) + end + + @doc """ + Holds every request for which `match.(method, params)` is true until `release/1`. The test + process is told `{:held, pid, method, params}` when one arrives. + """ + def hold(%{agent: agent}, match) when is_function(match, 2) do + watcher = self() + Agent.update(agent, &%{&1 | hold: match, watcher: watcher}) + end + + @doc "Releases a held request." + def release(pid), do: send(pid, :release) + + @doc "Stops holding requests." + def unhold(%{agent: agent}), do: Agent.update(agent, &%{&1 | hold: nil}) + + @doc "Every request so far, oldest first: `%{method:, params:, at:}` with `at` in monotonic ms." + def requests(%{agent: agent}), do: agent |> Agent.get(& &1.requests) |> Enum.reverse() + + @doc "The requests of one method." + def requests(fake, method), do: fake |> requests() |> Enum.filter(&(&1.method == method)) + + @doc "The text each message in a chat shows now, in the order the messages were sent." + def chat(%{agent: agent}, chat_id) do + chat_id = to_string(chat_id) + + agent + |> Agent.get(& &1.messages) + |> Enum.filter(fn {_id, m} -> to_string(m.chat_id) == chat_id end) + |> Enum.sort_by(fn {id, _} -> id end) + |> Enum.map(fn {_id, m} -> m.text end) + end + + @doc "The first message, by id, for which `fun` is true, as `{message_id, message}`." + def find_message(%{agent: agent}, fun) do + agent + |> Agent.get(& &1.messages) + |> Enum.sort_by(fn {id, _} -> id end) + |> Enum.find(fn {_id, m} -> fun.(m) end) + end + + @doc "A message as it stands (text, reply markup)." + def message(%{agent: agent}, message_id), + do: Agent.get(agent, &Map.get(&1.messages, message_id)) + + ## The plug + + @impl Plug + def init(agent), do: agent + + @impl Plug + def call(conn, agent) do + case String.split(conn.request_path, "/", trim: true) do + ["bot" <> token, method] -> api(conn, agent, token, method) + ["file", "bot" <> token | path] -> file(conn, agent, token, Enum.join(path, "/")) + _ -> send_resp(conn, 404, "") + end + end + + defp api(conn, agent, token, method) do + {:ok, body, conn} = read_body(conn) + params = if body == "", do: %{}, else: Jason.decode!(body) + + Agent.update( + agent, + &%{&1 | requests: [%{method: method, params: params, at: now()} | &1.requests]} + ) + + maybe_hold(agent, method, params) + + cond do + token != @token -> + json(conn, 401, %{"ok" => false, "error_code" => 401, "description" => "Unauthorized"}) + + failure = take_failure(agent, method) -> + {code, description} = failure + json(conn, code, %{"ok" => false, "error_code" => code, "description" => description}) + + true -> + case handle(method, params, agent) do + {:ok, result} -> + json(conn, 200, %{"ok" => true, "result" => result}) + + {:error, code, desc} -> + json(conn, code, %{"ok" => false, "error_code" => code, "description" => desc}) + end + end + end + + defp file(conn, agent, token, path) do + file = + agent + |> Agent.get(& &1.files) + |> Enum.find_value(fn {_id, f} -> if f.path == path, do: f end) + + cond do + token != @token -> send_resp(conn, 401, "") + file == nil -> send_resp(conn, 404, "") + true -> send_resp(conn, 200, file.bytes) + end + end + + defp maybe_hold(agent, method, params) do + %{hold: match, watcher: watcher} = Agent.get(agent, & &1) + + if match && match.(method, params) do + send(watcher, {:held, self(), method, params}) + + receive do + :release -> :ok + after + 30_000 -> :ok + end + end + end + + defp take_failure(agent, method) do + Agent.get_and_update(agent, fn state -> + case Map.get(state.fail, method) do + {n, code, description} when n > 0 -> + {{code, description}, put_in(state.fail[method], {n - 1, code, description})} + + _ -> + {nil, state} + end + end) + end + + ## Methods + + defp handle("getMe", _params, agent), do: {:ok, Agent.get(agent, & &1.me)} + + defp handle("getUpdates", params, agent) do + offset = Map.get(params, "offset") + timeout_ms = min(Map.get(params, "timeout", 0), 2) * 1_000 + deadline = now() + timeout_ms + await_updates(agent, offset, deadline) + end + + defp handle("sendMessage", %{"chat_id" => chat, "text" => text} = params, agent) do + with :ok <- valid_text(text, params) do + Agent.get_and_update(agent, fn state -> + id = state.next_message + message = %{chat_id: chat, text: text, reply_markup: Map.get(params, "reply_markup")} + + {{:ok, %{"message_id" => id, "chat" => %{"id" => chat}, "text" => text}}, + %{state | next_message: id + 1, messages: Map.put(state.messages, id, message)}} + end) + end + end + + defp handle("editMessageText", %{"message_id" => id, "text" => text} = params, agent) do + with :ok <- valid_text(text, params) do + Agent.get_and_update(agent, &edit(&1, id, text)) + end + end + + defp handle("editMessageReplyMarkup", %{"message_id" => id} = params, agent) do + Agent.update(agent, fn state -> + case Map.get(state.messages, id) do + nil -> state + message -> put_in(state.messages[id], %{message | reply_markup: params["reply_markup"]}) + end + end) + + {:ok, true} + end + + defp handle("getFile", %{"file_id" => file_id}, agent) do + case Agent.get(agent, &Map.get(&1.files, file_id)) do + nil -> + {:error, 400, "Bad Request: invalid file_id"} + + file -> + {:ok, + %{"file_id" => file_id, "file_path" => file.path, "file_size" => byte_size(file.bytes)}} + end + end + + defp handle(method, _params, _agent) + when method in ["sendChatAction", "answerCallbackQuery", "deleteWebhook"], + do: {:ok, true} + + defp handle(method, _params, _agent), do: {:error, 404, "Not Found: method #{method}"} + + defp edit(state, id, text) do + case Map.get(state.messages, id) do + nil -> + {{:error, 400, "Bad Request: message to edit not found"}, state} + + %{text: ^text} -> + {{:error, 400, "Bad Request: message is not modified"}, state} + + message -> + {{:ok, %{"message_id" => id, "text" => text}}, + put_in(state.messages[id], %{message | text: text})} + end + end + + defp await_updates(agent, offset, deadline) do + pending = + Agent.get_and_update(agent, fn state -> + # An offset confirms every update below it, as Telegram does; the rest stay until confirmed. + kept = + if offset, + do: Enum.filter(state.updates, &(&1["update_id"] >= offset)), + else: state.updates + + {kept, %{state | updates: kept}} + end) + + cond do + pending != [] -> + {:ok, pending} + + now() >= deadline -> + {:ok, []} + + true -> + Process.sleep(20) + await_updates(agent, offset, deadline) + end + end + + defp valid_text(text, params) do + cond do + String.trim(text) == "" -> + {:error, 400, "Bad Request: message text is empty"} + + utf16_length(text) > 4096 -> + {:error, 400, "Bad Request: message is too long"} + + params["parse_mode"] == "MarkdownV2" -> + case MarkdownV2.check(text) do + :ok -> + :ok + + {:error, reason} -> + {:error, 400, "Bad Request: can't parse entities: #{inspect(reason)}"} + end + + true -> + :ok + end + end + + defp json(conn, status, body) do + conn |> put_resp_content_type("application/json") |> send_resp(status, Jason.encode!(body)) + end + + # Telegram counts UTF-16 code units; written here rather than borrowed from the adapter, so the + # fake does not measure the adapter with the adapter's own ruler. + defp utf16_length(text), + do: for(<>, reduce: 0, do: (n -> n + if(c > 0xFFFF, do: 2, else: 1))) + + defp now, do: System.monotonic_time(:millisecond) +end diff --git a/test/support/telegram/markdown_v2.ex b/test/support/telegram/markdown_v2.ex new file mode 100644 index 0000000..fe5a228 --- /dev/null +++ b/test/support/telegram/markdown_v2.ex @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.FakeTelegram.MarkdownV2 do + @moduledoc """ + Telegram's MarkdownV2 parse rules, as a checker the suite and the fake Bot API share (slice 071). + + Written from the Bot API's "MarkdownV2 style" section, and stricter than Telegram where the + documentation is silent, so a text this accepts is one Telegram accepts and not the other way + round: + + - outside an entity, each of `_ * [ ] ( ) ~ ` > # + - = | { } . !` is markup or must be escaped + with `\\`, and `\\` itself must be escaped; `>` is markup only at the start of a line; + - `\\` may escape any character with a code from 1 to 126, and nothing else; + - inside `code` and `pre`, an unescaped `` ` `` ends the entity and `\\` must escape; + - inside a link's `(...)`, an unescaped `)` ends it and `\\` must escape; + - `__` is underline and is read before `_` (Telegram's greedy rule); `||` is a spoiler; + - entities nest properly and none is left open at the end. + + `check/1` answers `:ok` or the first violation with its byte offset. + """ + + @reserved ~c"_*[]()~`>#+-=|{}.!" + + @doc "Whether Telegram would parse `text` as MarkdownV2, and the first reason it would not." + @spec check(String.t()) :: :ok | {:error, {atom(), term(), non_neg_integer()}} + def check(text) when is_binary(text), do: run(text, [], true, 0) + + defp run("", [], _line_start, _pos), do: :ok + defp run("", stack, _line_start, pos), do: {:error, {:unclosed, stack, pos}} + + defp run(<<"\\", c::utf8, rest::binary>>, stack, _ls, pos) when c in 1..126, + do: run(rest, stack, false, pos + 2) + + defp run(<<"\\", rest::binary>>, _stack, _ls, pos), + do: {:error, {:bad_escape, String.slice(rest, 0, 1), pos}} + + defp run(<<"```", rest::binary>>, stack, _ls, pos) do + case verbatim(skip_language(rest), "```", pos + 3) do + {:ok, rest, pos} -> run(rest, stack, false, pos) + error -> error + end + end + + defp run(<<"`", rest::binary>>, stack, _ls, pos) do + case verbatim(rest, "`", pos + 1) do + {:ok, rest, pos} -> run(rest, stack, false, pos) + error -> error + end + end + + defp run(<<">", rest::binary>>, stack, true, pos), do: run(rest, stack, false, pos + 1) + defp run(<<"||", rest::binary>>, stack, _ls, pos), do: toggle(:spoiler, rest, stack, pos, 2) + defp run(<<"__", rest::binary>>, stack, _ls, pos), do: toggle(:underline, rest, stack, pos, 2) + defp run(<<"_", rest::binary>>, stack, _ls, pos), do: toggle(:italic, rest, stack, pos, 1) + defp run(<<"*", rest::binary>>, stack, _ls, pos), do: toggle(:bold, rest, stack, pos, 1) + defp run(<<"~", rest::binary>>, stack, _ls, pos), do: toggle(:strike, rest, stack, pos, 1) + defp run(<<"[", rest::binary>>, stack, _ls, pos), do: run(rest, [:link | stack], false, pos + 1) + + defp run(<<"](", rest::binary>>, [:link | stack], _ls, pos) do + case url(rest, pos + 2) do + {:ok, rest, pos} -> run(rest, stack, false, pos) + error -> error + end + end + + defp run(<<"\n", rest::binary>>, stack, _ls, pos), do: run(rest, stack, true, pos + 1) + + defp run(<>, _stack, _ls, pos) when c in @reserved, + do: {:error, {:unescaped, <>, pos}} + + defp run(<>, stack, _ls, pos), + do: run(rest, stack, false, pos + byte_size(<>)) + + defp run(<>, _stack, _ls, pos), do: {:error, {:not_utf8, byte, pos}} + + defp toggle(kind, rest, [kind | stack], pos, len), do: run(rest, stack, false, pos + len) + + defp toggle(kind, rest, stack, pos, len) do + if kind in stack, + do: {:error, {:improper_nesting, kind, pos}}, + else: run(rest, [kind | stack], false, pos + len) + end + + # A pre block's language runs to the first newline; it may be absent. + defp skip_language(rest) do + case String.split(rest, "\n", parts: 2) do + [lang, body] -> if Regex.match?(~r/\A[^`\s\\]*\z/, lang), do: body, else: rest + [_] -> rest + end + end + + # Inside code or pre: `\\` escapes, the closing delimiter ends it, everything else is text. + defp verbatim(<<"\\", c::utf8, rest::binary>>, close, pos) when c in 1..126, + do: verbatim(rest, close, pos + 2) + + defp verbatim(<<"\\", _::binary>>, _close, pos), do: {:error, {:bad_escape_in_code, nil, pos}} + + defp verbatim(text, close, pos) do + cond do + text == "" -> + {:error, {:unclosed, [:code], pos}} + + String.starts_with?(text, close) -> + {:ok, binary_part(text, byte_size(close), byte_size(text) - byte_size(close)), + pos + byte_size(close)} + + String.starts_with?(text, "`") -> + {:error, {:unescaped_in_code, "`", pos}} + + true -> + <> = text + verbatim(rest, close, pos + byte_size(<>)) + end + end + + defp url(<<"\\", c::utf8, rest::binary>>, pos) when c in 1..126, do: url(rest, pos + 2) + defp url(<<"\\", _::binary>>, pos), do: {:error, {:bad_escape_in_url, nil, pos}} + defp url(<<")", rest::binary>>, pos), do: {:ok, rest, pos + 1} + defp url("", pos), do: {:error, {:unclosed, [:url], pos}} + defp url(<>, pos), do: url(rest, pos + byte_size(<>)) +end diff --git a/test/trinity/gateways/approvals_test.exs b/test/trinity/gateways/approvals_test.exs index 930132d..833599c 100644 --- a/test/trinity/gateways/approvals_test.exs +++ b/test/trinity/gateways/approvals_test.exs @@ -158,6 +158,35 @@ defmodule Trinity.Gateways.ApprovalsTest do refute Cap.allows?(@adapter, nil) end + # Slice 071, finding F2. Approval ids are UUIDv7, whose first 48 bits are the time in + # milliseconds, so the eight-character short form a person is shown is the same for every request + # raised within the same 65 seconds. `/approve ` then decided whichever pending request + # came first, which need not be the one the person was looking at. + test "a short id two pending requests share is refused as ambiguous, and neither is decided" do + session_id = session_for_conversation() + + {:ok, %Approval{id: first}} = + Permissions.request_approval(session_id, "write_note", %{"path" => "a.md"}, risk: :write) + + {:ok, %Approval{id: second}} = + Permissions.request_approval(session_id, "write_note", %{"path" => "b.md"}, risk: :write) + + short = String.slice(first, 0, 8) + + assert String.starts_with?(second, short), + "the two ids no longer share a prefix: #{first} #{second}" + + assert {:ok, :command} = Router.inbound(@adapter, @conv, @user, "/approve #{short}") + assert shown() =~ "matches 2 pending requests" + assert %Approval{status: "pending"} = Permissions.get_approval(first) + assert %Approval{status: "pending"} = Permissions.get_approval(second) + + # The full id is never ambiguous, and is what a button carries. + assert {:ok, :command} = Router.inbound(@adapter, @conv, @user, "/approve #{second}") + assert %Approval{status: "allowed"} = Permissions.get_approval(second) + assert %Approval{status: "pending"} = Permissions.get_approval(first) + end + test "a raised request in someone else's session is not rendered into this channel" do _mine = session_for_conversation() diff --git a/test/trinity/gateways/console_test.exs b/test/trinity/gateways/console_test.exs index 756cad1..498da10 100644 --- a/test/trinity/gateways/console_test.exs +++ b/test/trinity/gateways/console_test.exs @@ -17,7 +17,9 @@ defmodule Trinity.Gateways.ConsoleTest do end test "it implements every callback the behaviour declares" do - callbacks = Adapter.behaviour_info(:callbacks) |> Enum.sort() + # Optional callbacks (slice 071's `status/0`) are the adapter's to offer, not to owe. + optional = Adapter.behaviour_info(:optional_callbacks) + callbacks = (Adapter.behaviour_info(:callbacks) -- optional) |> Enum.sort() exported = Console.__info__(:functions) for {name, arity} <- callbacks do diff --git a/test/trinity/gateways/router_test.exs b/test/trinity/gateways/router_test.exs index 50019c4..d38d083 100644 --- a/test/trinity/gateways/router_test.exs +++ b/test/trinity/gateways/router_test.exs @@ -54,8 +54,11 @@ defmodule Trinity.Gateways.RouterTest do [shown] = Console.text(@conv) {:ok, identity, _} = Identities.admit("console", @user) - assert shown =~ identity.code assert shown =~ "pair" + # Slice 071, finding F1: the code is shown on /gateways and nowhere else (docs/07). A prompt + # that carried it would let anyone who can write to the channel pair by echoing it back, + # which on a public messaging platform is anyone who finds the bot. + refute shown =~ identity.code # No session was created and the model was never called for someone not let in. assert Router.session_of(@adapter, @conv) == nil diff --git a/test/trinity/gateways/supervisor_test.exs b/test/trinity/gateways/supervisor_test.exs new file mode 100644 index 0000000..54f38cc --- /dev/null +++ b/test/trinity/gateways/supervisor_test.exs @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Gateways.SupervisorTest do + @moduledoc """ + Slice 071: the gateways run in the application's own tree. The router comes first, then each + adapter in force; with none configured there is no child at all, which is the suite's case and a + default desktop's. `TRINITY_GATEWAYS` names adapters from the `available:` list, and a name + that is not on it matches nothing. + """ + use ExUnit.Case, async: false + + alias Trinity.Gateways + alias Trinity.Gateways.{Console, Router, Telegram} + + setup do + previous = Application.get_env(:trinity, :gateways) + on_exit(fn -> Application.put_env(:trinity, :gateways, previous || []) end) + :ok + end + + test "none configured, no children; otherwise the router first, then each adapter" do + assert Gateways.Supervisor.children([]) == [] + assert Gateways.Supervisor.children([Telegram]) == [Router, {Telegram, []}] + end + + test "the application runs the gateways' supervisor, and in the suite it has nothing to run" do + pid = Process.whereis(Gateways.Supervisor) + assert is_pid(pid) + assert Supervisor.which_children(pid) == [] + end + + test "enabled names pick from the available modules, and an unknown name picks nothing" do + Application.put_env(:trinity, :gateways, + available: [Console, Telegram], + enabled: ["telegram", "no_such_gateway"] + ) + + assert Gateways.adapters() == [Telegram] + + Application.put_env(:trinity, :gateways, + adapters: [Console], + available: [Console, Telegram], + enabled: ["console", "telegram"] + ) + + assert Gateways.adapters() == [Console, Telegram] + end + + test "the configuration the build ships offers Telegram and runs nothing until it is named" do + {config, _} = Config.Reader.read_imports!("config/config.exs", env: :prod, target: :host) + gateways = config |> Keyword.fetch!(:trinity) |> Keyword.fetch!(:gateways) + assert Telegram in Keyword.fetch!(gateways, :available) + refute Keyword.has_key?(gateways, :adapters) + refute Keyword.has_key?(gateways, :enabled) + end + + test "TRINITY_GATEWAYS becomes the enabled names, trimmed, in a non-test environment" do + previous = System.get_env("TRINITY_GATEWAYS") + System.put_env("TRINITY_GATEWAYS", " telegram , console,") + + on_exit(fn -> + if previous, + do: System.put_env("TRINITY_GATEWAYS", previous), + else: System.delete_env("TRINITY_GATEWAYS") + end) + + config = Config.Reader.read!("config/runtime.exs", env: :dev, target: :host) + assert config[:trinity][:gateways][:enabled] == ["telegram", "console"] + + System.delete_env("TRINITY_GATEWAYS") + config = Config.Reader.read!("config/runtime.exs", env: :dev, target: :host) + refute Keyword.has_key?(config[:trinity][:gateways] || [], :enabled) + end +end diff --git a/test/trinity/gateways/telegram/markdown_test.exs b/test/trinity/gateways/telegram/markdown_test.exs new file mode 100644 index 0000000..52c9561 --- /dev/null +++ b/test/trinity/gateways/telegram/markdown_test.exs @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Gateways.Telegram.MarkdownTest do + @moduledoc """ + Slice 071, AC4: formatting, escaping and chunking with tricky markdown (code blocks, + underscores, links), checked against the MarkdownV2 parse rules in + `Trinity.FakeTelegram.MarkdownV2`, the same checker the fake Bot API refuses a message with. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Trinity.FakeTelegram.MarkdownV2 + alias Trinity.Gateways.Telegram + alias Trinity.Gateways.Telegram.Markdown + + @caps Telegram.capabilities() + + defp v2(text), do: Markdown.to_markdown_v2(text) + + describe "the checker discriminates" do + test "it refuses what Telegram refuses" do + assert {:error, {:unescaped, ".", _}} = MarkdownV2.check("a.b") + assert {:error, {:unclosed, [:bold], _}} = MarkdownV2.check("*open") + assert {:error, {:unescaped, ")", _}} = MarkdownV2.check("[a](http://x.y/a)b)") + assert {:error, {:unescaped, ">", _}} = MarkdownV2.check("a > b") + assert {:error, {:improper_nesting, :bold, _}} = MarkdownV2.check("*a _b *c* d_ e*") + assert :ok = MarkdownV2.check("*bold* _it_ `co\\`de` [l](http://x\\)y) \\. \\>") + assert :ok = MarkdownV2.check(">quote\n```elixir\nIO.puts(\"*_\")\n```") + end + + test "the plain-text formatter the console uses is not MarkdownV2, which is why this module exists" do + text = "Done. See `mix test` (it passed) and file_name.ex!" + [chunk] = Trinity.Gateways.Format.format(text, %{markdown: true, max_length: 4096}) + assert {:error, _} = MarkdownV2.check(chunk) + assert :ok = MarkdownV2.check(v2(text)) + end + end + + describe "AC4: escaping" do + test "every reserved character outside an entity is escaped" do + assert v2("_*[]()~`>#+-=|{}.!\\") == + "\\_\\*\\[\\]\\(\\)\\~\\`\\>\\#\\+\\-\\=\\|\\{\\}\\.\\!\\\\" + end + + test "underscores inside a word are characters, not italics" do + assert v2("call snake_case_name and __init__ here") == + "call snake\\_case\\_name and *init* here" + + assert v2("my_var_") == "my\\_var\\_" + assert v2("_real italic_ then x_y") == "_real italic_ then x\\_y" + end + + test "a markdown backslash escape is the character, escaped once" do + assert v2("\\*not bold\\*") == "\\*not bold\\*" + assert v2("1\\. not a list") == "1\\. not a list" + end + + test "text that is not UTF-8 becomes the replacement character, not a refused message" do + assert :ok = MarkdownV2.check(v2(<<"ok ", 0xFF, " fine.">>)) + end + end + + describe "AC4: entities" do + test "bold, italic, strike and inline code" do + assert v2("**bold** *it* ~~gone~~ `x.y`") == "*bold* _it_ ~gone~ `x.y`" + end + + test "inline code escapes only the backtick and the backslash" do + assert v2("run `a\\b` and ``a`b``") == "run `a\\\\b` and `a\\`b`" + end + + test "a fenced block keeps its language and escapes only ` and \\ inside" do + text = "Before:\n```elixir\ndef f(x), do: x * 2 # ok.\nIO.puts(\"a\\\\b `c`\")\n```\nAfter." + + assert v2(text) == + "Before:\n```elixir\ndef f(x), do: x * 2 # ok.\nIO.puts(\"a\\\\\\\\b \\`c\\`\")\n```\nAfter\\." + + assert :ok = MarkdownV2.check(v2(text)) + end + + test "a fence that never closes is still code, and is closed" do + assert v2("```\nhalf a stream (") == "```\nhalf a stream (\n```" + end + + test "a language that is not a plain word is dropped rather than risked" do + assert v2("```c++ {x}\nint a;\n```") == "```\nint a;\n```" + end + + test "links: the label is escaped, the URL escapes ) and \\, a paren pair in the URL survives" do + assert v2("[the docs](https://example.com/a_b.html)") == + "[the docs](https://example.com/a_b.html)" + + assert v2("see [Foo (bar)](https://en.wikipedia.org/wiki/Foo_(bar)).") == + "see [Foo \\(bar\\)](https://en.wikipedia.org/wiki/Foo_(bar\\))\\." + + assert :ok = + MarkdownV2.check(v2("see [Foo (bar)](https://en.wikipedia.org/wiki/Foo_(bar)).")) + end + + test "a link to anything but http, https, tg or mailto stays text" do + assert v2("[x](javascript:alert(1))") == "\\[x\\]\\(javascript:alert\\(1\\)\\)" + assert v2("[x](relative/path)") == "\\[x\\]\\(relative/path\\)" + end + + test "headings are bold, lists are bullets, numbers keep an escaped dot, quotes are quotes" do + text = "# Title **here**\n- one\n* two\n3. three\n> said _so_\n---" + assert v2(text) == "*Title here*\n• one\n• two\n3\\. three\n>said _so_\n──────────" + assert :ok = MarkdownV2.check(v2(text)) + end + + test "two entities of a kind side by side do not become an underline" do + out = v2("*a*_b_") + assert :ok = MarkdownV2.check(out) + refute out =~ "__" + end + end + + describe "AC4: half-written markdown, as a stream delivers it" do + test "an entity that is not closed yet is escaped text" do + for partial <- ["**bol", "*it", "`co", "[link](https://exa", "~~str", "_x", "```py\nx = 1"] do + assert :ok = MarkdownV2.check(v2(partial)), "#{inspect(partial)} -> #{v2(partial)}" + end + + assert v2("**bol") == "\\*\\*bol" + assert v2("[link](https://exa") == "\\[link\\]\\(https://exa" + end + end + + describe "AC4: chunking at 4096" do + test "a long answer splits into chunks that each fit and each parse" do + paragraph = "A sentence with **bold**, `code` and a [link](https://example.com). " + text = String.duplicate(paragraph, 200) + chunks = Telegram.format(text, @caps) + + assert length(chunks) > 1 + + for chunk <- chunks do + out = v2(chunk) + assert Markdown.utf16_length(out) <= 4096 + assert :ok = MarkdownV2.check(out) + end + end + + test "escaping can double a text's length, and the chunks are sized after it" do + text = String.duplicate(".", 6000) + chunks = Telegram.format(text, @caps) + assert Enum.all?(chunks, &(Markdown.utf16_length(v2(&1)) <= 4096)) + assert Enum.join(chunks) == text + end + + test "emoji count two units each, as Telegram counts them" do + assert Markdown.utf16_length("a😀") == 3 + text = String.duplicate("😀", 3000) + chunks = Telegram.format(text, @caps) + assert Enum.all?(chunks, &(Markdown.utf16_length(v2(&1)) <= 4096)) + assert Enum.join(chunks) == text + end + + test "a code block split across chunks is closed and reopened, and each chunk parses" do + code = Enum.map_join(1..900, "\n", &"line_#{&1} = #{&1} * 2.0") + text = "Here:\n```python\n" <> code <> "\n```\nDone." + chunks = Telegram.format(text, @caps) + assert length(chunks) > 1 + + for chunk <- chunks do + out = v2(chunk) + assert :ok = MarkdownV2.check(out) + assert Markdown.utf16_length(out) <= 4096 + end + end + end + + describe "AC4: the property, over generated input" do + # - + = | { } . ! > \\ \n \n\n https://x.y/(z) word snake_case 😀 é) + @pieces ~w(** * __ _ ~~ ` ``` [ ] ( ) + @pieces @pieces ++ [" ", " ", "[a](https://e.x/p_(q))", "1. ", "- ", "> ", "```py\n"] + + property "whatever the markdown, the output parses as MarkdownV2" do + check all(parts <- list_of(member_of(@pieces), max_length: 60), max_runs: 1_000) do + text = Enum.join(parts) + out = v2(text) + assert :ok == MarkdownV2.check(out), "input #{inspect(text)}\noutput #{inspect(out)}" + end + end + + property "whatever the markdown, every chunk fits 4096 units and parses" do + check all( + parts <- list_of(member_of(@pieces), min_length: 1, max_length: 40), + times <- integer(1..400), + max_runs: 60 + ) do + text = parts |> Enum.join() |> String.duplicate(times) + + for chunk <- Telegram.format(text, @caps) do + out = v2(chunk) + assert Markdown.utf16_length(out) <= 4096 + assert :ok == MarkdownV2.check(out), "chunk #{inspect(chunk)}" + end + end + end + end +end diff --git a/test/trinity/gateways/telegram_test.exs b/test/trinity/gateways/telegram_test.exs new file mode 100644 index 0000000..9193627 --- /dev/null +++ b/test/trinity/gateways/telegram_test.exs @@ -0,0 +1,485 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Gateways.TelegramTest do + @moduledoc """ + Slice 071 against a fake Bot API on a loopback port (`Trinity.FakeTelegram.BotApi`), with the real + router, the real sessions and the fake model: the automatic halves of AC1 (a DM answered by a + reply edited in place, edits at least a second apart, ending in the final text), AC2 (the inline + keyboard round trip, and no buttons above the channel's ceiling), AC3 (a photo stored and handed + to the model as an image), AC6 (a scheduled run delivered to a chat), and AC5 (the poller killed + mid-update, restarted, and the update not processed twice). Also: `/start` pairing, groups, and + the token absent from every log line. + """ + use Trinity.SessionCase + + import ExUnit.CaptureLog + + alias Trinity.FakeTelegram.BotApi + alias Trinity.Gateways.{Identities, Router, Telegram} + alias Trinity.Gateways.Telegram.{Markdown, Offset, Outbox, Poller} + alias Trinity.LLM.Providers.Fake + alias Trinity.Permissions + alias Trinity.Permissions.Approval + + @user 4242 + + setup do + fake = BotApi.start!() + start_supervised!(Router) + start_supervised!(Telegram) + await(fn -> Outbox.status().state == :running end) + {:ok, fake: fake} + end + + ## Helpers + + defp pair!(user) do + {:ok, identity, :pending} = Identities.admit("telegram", to_string(user)) + {:ok, _} = Identities.pair("telegram", to_string(user), identity.code) + :ok + end + + defp await(fun, timeout \\ 8_000) do + deadline = System.monotonic_time(:millisecond) + timeout + do_await(fun, deadline) + end + + defp do_await(fun, deadline) do + case fun.() do + falsy when falsy in [nil, false] -> + if System.monotonic_time(:millisecond) > deadline do + flunk("never became truthy") + else + Process.sleep(25) + do_await(fun, deadline) + end + + value -> + value + end + end + + defp shown(fake, chat), do: fake |> BotApi.chat(chat) |> Enum.join("\n") + defp await_shown(fake, chat, fragment), do: await(fn -> shown(fake, chat) =~ fragment end) + + # The words of `text` as deltas with a pause after each, so the stream is long enough to be + # flushed and edited several times; the deltas join back to exactly `text`. + defp deltas(text, sleep_ms) do + [first | rest] = String.split(text, " ") + + [first | Enum.map(rest, &(" " <> &1))] + |> Enum.flat_map(&[{:text_delta, &1}, {:sleep, sleep_ms}]) + |> Kernel.++([{:usage, %{input_tokens: 5, output_tokens: 5}}, {:done, :stop}]) + end + + ## AC1, automatic half + + test "AC1: a DM is answered by one message edited in place, at most once a second, ending in the final text", + %{fake: fake} do + pair!(@user) + words = Enum.map_join(1..14, " ", &"word#{&1}.") + Fake.scripts([deltas(words, 250)]) + + BotApi.push(fake, BotApi.text_message(@user, "hello")) + + final = String.trim(words) + await(fn -> BotApi.chat(fake, @user) == [Markdown.to_markdown_v2(final)] end, 15_000) + + # The typing indicator went out before the first message did. + [first_typing | _] = BotApi.requests(fake, "sendChatAction") + [first_send | _] = BotApi.requests(fake, "sendMessage") + assert first_typing.at <= first_send.at + + # One message, edited several times while the turn streamed. + assert [%{params: %{"text" => partial}}] = BotApi.requests(fake, "sendMessage") + assert String.length(partial) < String.length(Markdown.to_markdown_v2(final)) + edits = BotApi.requests(fake, "editMessageText") + assert length(edits) >= 2 + + # Never two edits of a message within a second of each other, including the first after the send. + times = Enum.map([first_send | edits], & &1.at) + + # SLICE.md: "edit-throttle >= 1 s". The floor is the spec's number, not the module's constant, + # so a module that lowered its constant would fail here rather than pass against itself. Five + # milliseconds of slack is the fake's own timestamping, taken after the request is read. + for {a, b} <- Enum.zip(times, tl(times)) do + assert b - a >= 1_000 - 5, "edits #{b - a} ms apart" + end + + assert Outbox.edit_interval_ms() >= 1_000 + + # The last thing sent to that message is the whole answer. + assert List.last(edits).params["text"] == Markdown.to_markdown_v2(final) + end + + test "a reply in MarkdownV2 that Telegram refuses to parse is sent again as plain text", %{ + fake: fake + } do + pair!(@user) + BotApi.fail(fake, "sendMessage", 1, 400, "Bad Request: can't parse entities: test") + BotApi.push(fake, BotApi.text_message(@user, "/help")) + + await_shown(fake, @user, "What I answer here") + [refused, plain] = BotApi.requests(fake, "sendMessage") + assert refused.params["parse_mode"] == "MarkdownV2" + refute Map.has_key?(plain.params, "parse_mode") + end + + ## Pairing with /start + + test "/start from a stranger is answered with the pairing prompt, never the code; the code pairs", + %{fake: fake} do + BotApi.push(fake, BotApi.text_message(77, "/start")) + await_shown(fake, 77, "does not know you yet") + + identity = Identities.get("telegram", "77") + assert identity.state == "pending" + refute shown(fake, 77) =~ identity.code + assert Sessions.list_sessions() == [] + + BotApi.push(fake, BotApi.text_message(77, String.downcase(identity.code))) + await_shown(fake, 77, "Paired") + assert Identities.get("telegram", "77").state == "paired" + + BotApi.push(fake, BotApi.text_message(77, "/start")) + await_shown(fake, 77, "What I answer here") + end + + test "a t.me deep link's /start pairs in one message", %{fake: fake} do + {:ok, identity, :pending} = Identities.admit("telegram", "78") + BotApi.push(fake, BotApi.text_message(78, "/start " <> identity.code)) + await_shown(fake, 78, "Paired") + end + + ## Groups + + test "in a group the bot answers a mention or a command addressed to it, and reads nothing else", + %{fake: fake} do + pair!(@user) + group = %{"chat" => %{"id" => -100_555, "type" => "supergroup"}} + + BotApi.push(fake, BotApi.text_message(@user, "/help", group)) + BotApi.push(fake, BotApi.text_message(@user, "talking among ourselves", group)) + BotApi.push(fake, BotApi.text_message(@user, "/help@trinity_test_bot", group)) + + await_shown(fake, -100_555, "What I answer here") + Process.sleep(300) + assert length(BotApi.chat(fake, -100_555)) == 1 + + Fake.scripts([deltas("Yes, I am here.", 5)]) + BotApi.push(fake, BotApi.text_message(@user, "@trinity_test_bot are you there?", group)) + await_shown(fake, -100_555, "Yes, I am here") + + [session] = Sessions.list_sessions() + assert session.origin == "telegram" + assert session.origin_ref["conversation"] == "-100555" + [user_row | _] = Sessions.history(session.id) + assert user_row.content == "are you there?" + end + + test "messages from bots are ignored", %{fake: fake} do + BotApi.push(fake, BotApi.text_message(99, "hi", %{"from" => %{"id" => 99, "is_bot" => true}})) + Process.sleep(500) + assert BotApi.requests(fake, "sendMessage") == [] + assert Identities.get("telegram", "99") == nil + end + + ## AC2, automatic half + + defp session_with_turn(fake) do + pair!(@user) + Fake.scripts([deltas("ok", 5)]) + BotApi.push(fake, BotApi.text_message(@user, "start")) + await_shown(fake, @user, "ok") + await(fn -> Router.session_of(Telegram, to_string(@user)) end) + end + + defp keyboard(fake) do + Enum.find_value(BotApi.requests(fake, "sendMessage"), fn r -> + get_in(r.params, ["reply_markup", "inline_keyboard"]) + end) + end + + test "AC2: an approval carries Approve and Deny buttons; a press decides it as the presser", + %{fake: fake} do + session_id = session_with_turn(fake) + + {:ok, %Approval{id: id}} = + Permissions.request_approval(session_id, "write_note", %{"path" => "notes/a.md"}, + risk: :write + ) + + await_shown(fake, @user, "Approval needed: write\\_note") + [[approve, deny]] = keyboard(fake) + assert approve == %{"text" => "Approve", "callback_data" => "/approve " <> id} + assert deny == %{"text" => "Deny", "callback_data" => "/deny " <> id} + + {card, _} = BotApi.find_message(fake, &(&1.reply_markup != nil)) + + BotApi.push(fake, %{ + "callback_query" => %{ + "id" => "cb-1", + "from" => %{"id" => @user, "is_bot" => false, "first_name" => "Ayla"}, + "message" => %{"message_id" => card, "chat" => %{"id" => @user, "type" => "private"}}, + "data" => approve["callback_data"] + } + }) + + await(fn -> Permissions.get_approval(id).status == "allowed" end) + approval = Permissions.get_approval(id) + assert approval.decided_by == "gateway:telegram:#{@user}" + await_shown(fake, @user, "Approved") + + # The press was acknowledged, and the buttons came off the card. + assert [%{params: %{"callback_query_id" => "cb-1"}}] = + BotApi.requests(fake, "answerCallbackQuery") + + await(fn -> BotApi.message(fake, card).reply_markup == %{"inline_keyboard" => []} end) + end + + test "AC2: above the channel's ceiling there are no buttons, only where to decide it", %{ + fake: fake + } do + session_id = session_with_turn(fake) + + {:ok, %Approval{id: id}} = + Permissions.request_approval(session_id, "delete_tree", %{"path" => "/tmp/x"}, + risk: :destructive + ) + + await_shown(fake, @user, "may approve up to write") + assert keyboard(fake) == nil + + # A forged press is routed as typed text would be, so the cap refuses it the same way. + BotApi.push(fake, %{ + "callback_query" => %{ + "id" => "cb-2", + "from" => %{"id" => @user, "is_bot" => false}, + "message" => %{"message_id" => 1, "chat" => %{"id" => @user, "type" => "private"}}, + "data" => "/approve " <> id + } + }) + + await_shown(fake, @user, "That is a destructive request") + assert %Approval{status: "pending", decided_by: nil} = Permissions.get_approval(id) + end + + test "callback data that is not an approval command is acknowledged and ignored", %{fake: fake} do + pair!(@user) + + BotApi.push(fake, %{ + "callback_query" => %{ + "id" => "cb-3", + "from" => %{"id" => @user, "is_bot" => false}, + "message" => %{"message_id" => 1, "chat" => %{"id" => @user, "type" => "private"}}, + "data" => "/new" + } + }) + + await(fn -> BotApi.requests(fake, "answerCallbackQuery") != [] end) + Process.sleep(300) + assert BotApi.requests(fake, "sendMessage") == [] + end + + ## AC3, automatic half + + @jpeg <<0xFF, 0xD8, 0xFF, 0xE0>> <> :binary.copy("jpegbytes", 50) + + defp photo_update(user, file_id, caption) do + BotApi.text_message(user, nil, %{ + "photo" => [ + %{"file_id" => file_id <> "-small", "file_size" => 10, "width" => 90, "height" => 90}, + %{"file_id" => file_id, "file_size" => byte_size(@jpeg), "width" => 800, "height" => 600} + ], + "caption" => caption + }) + |> update_in(["message"], &Map.delete(&1, "text")) + end + + defp with_vision(fun) do + llm = Application.get_env(:trinity, :llm) + + models = + for m <- Keyword.fetch!(llm, :models), + do: if(m.id == "fake:chat", do: %{m | caps: m.caps ++ [:vision]}, else: m) + + Application.put_env(:trinity, :llm, Keyword.put(llm, :models, models)) + + try do + fun.() + after + Application.put_env(:trinity, :llm, llm) + end + end + + test "AC3: a photo is stored by its digest and reaches a vision model as an image", %{ + fake: fake + } do + pair!(@user) + BotApi.put_file(fake, "photo-1", @jpeg) + + with_vision(fn -> + Fake.scripts([deltas("A cat on a mat.", 5)]) + BotApi.push(fake, photo_update(@user, "photo-1", "what is this?")) + await_shown(fake, @user, "A cat on a mat") + + request = Fake.last_request() + user = Enum.find(request.messages, &(&1.role == "user")) + assert user.content == "what is this?" + assert [%{path: path, media_type: "image/jpeg"}] = user.images + assert File.read!(path) == @jpeg + + digest = :crypto.hash(:sha256, @jpeg) |> Base.encode16(case: :lower) + assert Path.basename(path) == digest <> ".jpg" + assert Path.dirname(path) == Path.join(fake.state_dir, "media") + + # The row records where it came from and what it was. + [session] = Sessions.list_sessions() + [row | _] = Sessions.history(session.id) + + assert [%{"digest" => "sha256:" <> ^digest, "origin" => "gateway:telegram"}] = + row.parts["images"] + end) + + # The largest size that fits was the one fetched. + assert [%{params: %{"file_id" => "photo-1"}}] = BotApi.requests(fake, "getFile") + end + + test "AC3: a model without vision is told an image was attached, and is not sent it", %{ + fake: fake + } do + pair!(@user) + BotApi.put_file(fake, "photo-2", @jpeg) + Fake.scripts([deltas("I cannot see images.", 5)]) + BotApi.push(fake, photo_update(@user, "photo-2", nil)) + await_shown(fake, @user, "I cannot see images") + + user = Enum.find(Fake.last_request().messages, &(&1.role == "user")) + refute Map.has_key?(user, :images) + assert user.content =~ "(image)" + assert user.content =~ "does not accept images" + end + + test "a stranger's photo is never downloaded: they get the pairing prompt and nothing is fetched", + %{fake: fake} do + BotApi.put_file(fake, "photo-3", @jpeg) + BotApi.push(fake, photo_update(555, "photo-3", "look")) + await_shown(fake, 555, "does not know you yet") + Process.sleep(200) + assert BotApi.requests(fake, "getFile") == [] + refute File.exists?(Path.join(fake.state_dir, "media")) + end + + test "a file that is not the image it claims to be is refused, and the text still goes", %{ + fake: fake + } do + pair!(@user) + BotApi.put_file(fake, "photo-4", "not a jpeg") + Fake.scripts([deltas("Noted.", 5)]) + BotApi.push(fake, photo_update(@user, "photo-4", "see this")) + + await_shown(fake, @user, "could not be read") + await_shown(fake, @user, "Noted") + user = Enum.find(Fake.last_request().messages, &(&1.role == "user")) + refute Map.has_key?(user, :images) + end + + ## AC5 + + test "AC5: the poller killed mid-update restarts, resumes, and does not process that update twice", + %{fake: fake} do + pair!(@user) + + # The router's answer to the third update is held at the fake, so the poller is inside that + # update (offset stored, update routed, reply not yet sent) when it is killed. + BotApi.hold(fake, fn method, params -> + method == "sendMessage" and params["text"] =~ "three" + end) + + for word <- ~w(one two three), do: BotApi.push(fake, BotApi.text_message(@user, "/" <> word)) + + assert_receive {:held, held, "sendMessage", _}, 8_000 + poller = Process.whereis(Poller) + ref = Process.monitor(poller) + Process.exit(poller, :kill) + assert_receive {:DOWN, ^ref, :process, ^poller, :killed} + + BotApi.unhold(fake) + BotApi.release(held) + + # The supervisor starts a new poller, which reads the stored offset and carries on. + restarted = await(fn -> (pid = Process.whereis(Poller)) && pid != poller && pid end) + assert is_pid(restarted) + BotApi.push(fake, BotApi.text_message(@user, "/four")) + await_shown(fake, @user, "/four") + Process.sleep(1_500) + + replies = BotApi.chat(fake, @user) + + for word <- ~w(one two three four) do + assert Enum.count(replies, &(&1 =~ "I do not know /#{word}")) == 1, + "/#{word} answered #{Enum.count(replies, &(&1 =~ "/#{word}"))} times: #{inspect(replies)}" + end + + assert Offset.load(fake.state_dir) == 5 + end + + ## The token, and the status + + test "the token appears in no log line, on the error paths included", %{fake: fake} do + pair!(@user) + BotApi.fail(fake, "sendMessage", 3, 500, "Internal Server Error") + + log = + capture_log(fn -> + BotApi.push(fake, BotApi.text_message(@user, "/help")) + await(fn -> BotApi.requests(fake, "sendMessage") != [] end) + Process.sleep(200) + + # A transport failure: the host is gone. The poller's next request fails and is logged. + Application.put_env( + :trinity, + :telegram, + Keyword.put(Application.get_env(:trinity, :telegram), :base_url, "http://127.0.0.1:1") + ) + + await(fn -> Outbox.status().state == :error end) + end) + + assert log =~ "telegram" + assert log =~ "econnrefused" + refute log =~ BotApi.token() + refute log =~ String.split(BotApi.token(), ":") |> List.last() + assert Outbox.status().detail =~ "cannot reach Telegram" + end + + test "status says which bot it polls as" do + assert %{state: :running, detail: "polling as @trinity_test_bot"} = Telegram.status() + end + + ## AC6, automatic half + + test "AC6: a scheduled run is delivered to a Telegram chat", %{fake: fake} do + Application.put_env(:trinity, :gateways, adapters: [Telegram]) + on_exit(fn -> Application.delete_env(:trinity, :gateways) end) + + {:ok, task} = + Trinity.Scheduler.create_task(%{ + name: "Morning check", + kind: "cron", + schedule: "0 9 * * *", + prompt: "summarise", + deliver_to: %{"kind" => "gateway", "adapter" => "telegram", "conversation" => "4242"} + }) + + {:ok, run} = Trinity.Scheduler.run_now(task) + + {:ok, run} = + Trinity.Scheduler.update_run(run, %{status: "ok", summary: "Three tasks are due."}) + + assert Trinity.Scheduler.Delivery.for(task) == Trinity.Scheduler.Delivery.Gateway + assert {:ok, delivered} = Trinity.Scheduler.Delivery.for(task).deliver(run, task) + assert delivered.delivered_at != nil + await_shown(fake, 4242, "Morning check: Three tasks are due\\.") + end +end diff --git a/test/trinity/llm/mapping_test.exs b/test/trinity/llm/mapping_test.exs index bb18d2c..168eabf 100644 --- a/test/trinity/llm/mapping_test.exs +++ b/test/trinity/llm/mapping_test.exs @@ -161,6 +161,52 @@ defmodule Trinity.LLM.Providers.ReqLLM.MappingTest do assert tool.role == :tool end + # Slice 071: a user message that came with images reaches req_llm as content parts, the text + # first, each image read from the file the gateway stored; a file that is gone is said in words. + test "a user message's images reach req_llm's context as image parts after the text" do + path = Path.join(System.tmp_dir!(), "mapping-image-#{System.unique_integer([:positive])}.jpg") + File.write!(path, <<0xFF, 0xD8, 0xFF, 0xE0, "pixels">>) + on_exit(fn -> File.rm(path) end) + gone = path <> ".missing" + + {:ok, request} = + Trinity.LLM.Request.new(%{ + messages: [ + %{ + role: "user", + content: "what is this?", + images: [ + %{path: path, media_type: "image/jpeg"}, + %{path: gone, media_type: "image/png"} + ] + } + ] + }) + + %ReqLLM.Context{messages: [user]} = Trinity.LLM.Providers.ReqLLM.context(request) + + assert [%{type: :text, text: text}, %{type: :image, data: data, media_type: "image/jpeg"}] = + user.content + + assert text =~ "what is this?" + assert text =~ "1 attached image(s) could not be read" + assert data == File.read!(path) + end + + test "images on anything but a user message, or not a path and a media type, are refused" do + assert {:error, {:invalid_request, {:images, _}}} = + Trinity.LLM.Request.new(%{ + messages: [ + %{role: "assistant", content: "x", images: [%{path: "a", media_type: "b"}]} + ] + }) + + assert {:error, {:invalid_request, {:images, _}}} = + Trinity.LLM.Request.new(%{ + messages: [%{role: "user", content: "x", images: ["a.jpg"]}] + }) + end + describe "classify/1" do test "an API error with a status classifies by status" do assert %Error{transient?: true, status: 429} = diff --git a/test/trinity/telemetry/redaction_test.exs b/test/trinity/telemetry/redaction_test.exs index 20a7b08..5f26b4d 100644 --- a/test/trinity/telemetry/redaction_test.exs +++ b/test/trinity/telemetry/redaction_test.exs @@ -36,6 +36,22 @@ defmodule Trinity.Telemetry.RedactionTest do refute Redaction.redact("token=#{jwt}") =~ "dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1g" end + # Slice 071: the Bot API puts the token in the URL path, `/bot:/`, where no + # prefix or key name announces it. The shape is the bot's numeric id, a colon, and the secret. + test "a Telegram bot token, alone or inside a Bot API URL" do + token = "071000001:TEST_fake_token_for_the_suite_only" + + for line <- [ + "token #{token}", + "POST https://api.telegram.org/bot#{token}/getUpdates failed", + "GET /file/bot#{token}/photos/a.jpg" + ] do + out = Redaction.redact(line) + refute out =~ "TEST_fake_token_for_the_suite_only", "not masked: #{line} -> #{out}" + assert out =~ "[REDACTED" + end + end + test "a named secret in a key/value position" do for line <- [ "password: hunter2-and-then-some", diff --git a/test/trinity_web/live/gateways_live_telegram_test.exs b/test/trinity_web/live/gateways_live_telegram_test.exs new file mode 100644 index 0000000..03e9873 --- /dev/null +++ b/test/trinity_web/live/gateways_live_telegram_test.exs @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule TrinityWeb.GatewaysLiveTelegramTest do + @moduledoc """ + Slice 071, "UI status": `/gateways` shows what the Telegram adapter is doing beside whether it + runs: which bot it polls as, idle because there is no token, or the last error. + """ + use TrinityWeb.ConnCase, async: false + + import Phoenix.LiveViewTest + + alias Trinity.FakeTelegram.BotApi + alias Trinity.Gateways.{Router, Telegram} + alias Trinity.Gateways.Telegram.Outbox + + setup do + Application.put_env(:trinity, :gateways, adapters: [Telegram]) + on_exit(fn -> Application.delete_env(:trinity, :gateways) end) + :ok + end + + defp await(fun, tries \\ 200) do + cond do + fun.() -> :ok + tries == 0 -> flunk("never became true") + true -> Process.sleep(25) && await(fun, tries - 1) + end + end + + test "the page says which bot the adapter polls as", %{conn: conn} do + _fake = BotApi.start!() + start_supervised!(Router) + start_supervised!(Telegram) + await(fn -> Outbox.status().state == :running end) + + {:ok, _view, html} = live(conn, ~p"/gateways") + assert html =~ "telegram" + assert html =~ "running" + assert html =~ "polling as @trinity_test_bot" + assert html =~ "4096" + end + + test "without a token the adapter is idle and the page says why", %{conn: conn} do + _fake = BotApi.start!(token: false) + previous = System.get_env("TELEGRAM_BOT_TOKEN") + System.delete_env("TELEGRAM_BOT_TOKEN") + on_exit(fn -> if previous, do: System.put_env("TELEGRAM_BOT_TOKEN", previous) end) + + start_supervised!(Router) + start_supervised!(Telegram) + + # The poller declined to start; the outbox runs and reports why. + assert Process.whereis(Trinity.Gateways.Telegram.Poller) == nil + + {:ok, _view, html} = live(conn, ~p"/gateways") + assert html =~ "idle: TELEGRAM_BOT_TOKEN is not set" + end +end