From 2e2397145a6593c97bd697f5d69ba3c4ea8eed92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zaj=C4=85c?= Date: Tue, 20 Oct 2026 17:56:23 +0000 Subject: [PATCH] Use the XDG base directories for Pi Pocket's files Pi Pocket kept everything in one folder, `~/.pi-pocket`: the database, people and their tokens, uploads, worktrees, extension drop-ins, the tunnel's log, and the browser's profile. That is not where any of them belong, and it put secrets beside caches. Each part now goes where the XDG base directory spec puts it: settings in `~/.config/pi-pocket`, the database and what sessions write in `~/.local/share/pi-pocket`, the tunnel's log in `~/.local/state/pi-pocket`, and the profile and downloaded cloudflared in `~/.cache/pi-pocket`. `XDG_*_HOME` moves them, and `PI_POCKET_CONFIG_DIR`, `PI_POCKET_DATA_DIR`, `PI_POCKET_STATE_DIR`, and `PI_POCKET_CACHE_DIR` name each one itself: the `--data` option and `PI_POCKET_DIR`, which named one folder, are gone. An older `~/.pi-pocket` moves into the new folders once (`dirs.ts`), entry by entry, and only where the destination has nothing: a migration cut off midway finishes on the next start, an unknown file stays where it is with a line saying so, a database already in the data folder is kept and named rather than silently passed over, and a Pi Pocket still running on the old folder stops the migration with a message instead of moving a live database. Files that older messages and isolated sessions name by their old absolute path are reported too, as they now live under the data folder. `host.dataDir` keeps meaning the data folder and no interface in `types/pocket.d.ts` changes; the lock stays in the data folder, since it guards that database and says which folder is taken. The NixOS module sets all four folders to `dataDir`, so an existing deployment keeps its files where they are. --- AGENTS.md | 2 +- README.md | 20 +- SECURITY.md | 2 +- docs/architecture.md | 2 +- docs/design/theming.md | 4 +- docs/examples/clock.ts | 2 +- docs/examples/no-force-push.ts | 2 +- docs/examples/server-status.ts | 2 +- docs/extensions.md | 2 +- docs/extensions/first-extension.md | 2 +- docs/extensions/policies/api-stability.md | 2 +- docs/extensions/policies/core-or-extension.md | 4 +- docs/extensions/reference/glossary.md | 2 +- docs/extensions/reference/host.md | 2 +- docs/extensions/themes.md | 4 +- docs/features.md | 10 +- docs/index.md | 22 +- docs/map.md | 11 +- nix/module.nix | 12 +- nix/tests/extension.ts | 2 +- nix/tests/installed.mjs | 34 +- site/architecture.html | 4 +- site/engineering.html | 15 +- site/index.html | 8 +- src/launcher/access.ts | 12 +- src/launcher/main.ts | 42 ++- src/launcher/options.ts | 14 +- src/server/app.ts | 42 ++- src/server/browser.ts | 6 +- src/server/browser/discovery.ts | 10 +- src/server/config.ts | 8 +- src/server/dirs.ts | 231 +++++++++++++ src/server/extensions/prompt.ts | 2 +- src/server/gallery.ts | 2 +- src/server/lock.ts | 10 +- src/server/main.ts | 39 ++- src/server/workspace.ts | 13 +- test/app.test.ts | 2 +- test/browser.test.ts | 4 +- test/composer-focus-ui.test.ts | 2 +- test/dirs.test.ts | 305 ++++++++++++++++++ test/extension-ui.test.ts | 2 +- test/extensions-hub-ui.test.ts | 2 +- test/helpers.ts | 20 +- test/launcher-ui.test.ts | 2 +- test/main.test.ts | 34 +- test/mobile-ui.test.ts | 2 +- test/model-scope.test.ts | 4 +- test/peeks-ui.test.ts | 2 +- test/pip-chat-ui.test.ts | 2 +- test/pip-models.test.ts | 3 +- test/qol.test.ts | 30 +- test/session-row-ui.test.ts | 2 +- test/sign-in-ui.test.ts | 15 +- test/signins.test.ts | 3 +- test/subagents-ui.test.ts | 2 +- 56 files changed, 846 insertions(+), 195 deletions(-) create mode 100644 src/server/dirs.ts create mode 100644 test/dirs.test.ts diff --git a/AGENTS.md b/AGENTS.md index 7e85c524..8cc6160f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,7 +5,7 @@ Pi Pocket is often edited from inside itself, while it runs. [docs/map.md](docs/ - `web/**` changes reload every open browser right away. Keep the app loadable after each save: a syntax error blanks the screen for everyone, including you. Check with `node --check web/.js` after editing. - `src/server/extensions/*.ts` changes are reinstalled into the running server. A file that fails to load keeps its previous version and reports the error. Each module default-exports `(host, web) => Extension | Extension[] | void`, possibly async (`web` adds routes and a browser module to the app: [docs/extensions/app.md](docs/extensions/app.md)). Start each module with a `/** … */` comment: its first sentence is the description in the app's Extensions sheet, where the owner can turn modules off (a module that is off is not loaded, even when edited). - Other `src/server/**` changes need a restart (menu → Restart server). Running work resumes after it, but a tool call cut off mid-run that is not replay-safe comes back as interrupted. Do not restart while your own tool call is running; finish the edit, run `npm run check` and `npm test`, then ask the user to restart. -- The owner's drop-in extensions in `~/.pi-pocket/extensions/` load the same way, after the built-in ones, and stay off until the owner turns them on. A `node_modules` link there lets them import Pi Pocket's packages. +- The owner's drop-in extensions in `~/.local/share/pi-pocket/extensions/` (`XDG_DATA_HOME` moves it) load the same way, after the built-in ones, and stay off until the owner turns them on. A `node_modules` link there lets them import Pi Pocket's packages. - Document kinds in `src/server/docs.ts` are stored data. Do not rename them, or change their scope, history, or fork settings: Pi Durable checks those against what is stored. - The server is plain TypeScript run by Node's type stripping: only erasable syntax (no enums, no parameter properties, no namespaces). Imports use `.ts` extensions. - Code style is Prettier (`.prettierrc.json`: 4 spaces, 100 columns) plus ESLint's layout rules (`eslint.config.js`): braces on every `if` and loop, and a blank line after declarations, around blocks, and before `return` and `throw`. Run `npm run format` after editing. Prettier leaves the markup in `html` templates alone, so lay it out by hand the same way: 4 spaces per level, and a tag that runs past 100 columns gets one attribute per line and its children on their own lines. In htm a line break is not a space: break only where there is no space, or between tags, never inside running text. diff --git a/README.md b/README.md index 61098c87..615d6a48 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ While it runs: **q** quit · **r** restart server · **a** change access · **o* - **Codemode.** On by default: Pi writes short scripts that call its tools, and every call still goes through the same checks. - **Lancet Guard.** Off by default; turn it on in Menu → Extensions. With the [specpi-lancet-guard](https://github.com/TannerMidd/SpecPi) Pi package installed and on, risky bash, write, and edit calls wait for someone in the session to approve. - **Roles, invites, and limits.** Steer or view-only access, to every session or just one, through one-time links and QR codes, with spend limits per person and per session. -- **Live-editable.** No build step: edit the web app or an extension and it reloads in place. Add your own extensions in `~/.pi-pocket/extensions/`: tools and checks for Pi, and pages, rail buttons, routes, and slash commands for the app. +- **Live-editable.** No build step: edit the web app or an extension and it reloads in place. Add your own extensions in `~/.local/share/pi-pocket/extensions/` (`XDG_DATA_HOME` moves it): tools and checks for Pi, and pages, rail buttons, routes, and slash commands for the app. The full tour is in [docs/features.md](docs/features.md). @@ -74,7 +74,7 @@ Choose a mode in the launcher, or pass `--access`: | `cloudflare` | Anyone with the link | Free quick tunnel with https, no account needed (Cloudflare offers quick tunnels for testing and development). The address changes each time the launcher starts but stays the same across server restarts. | | `tailscale` | Your tailnet | Tailscale address only, encrypted by the tailnet | -Cloudflare mode needs `cloudflared`. On Linux the launcher can download the official release into `~/.pi-pocket/bin`. To use another tunnel (ngrok, `tailscale serve`), choose `local` and point the tunnel at port 8787. +Cloudflare mode needs `cloudflared`. On Linux the launcher can download the official release into `~/.cache/pi-pocket/bin` (`XDG_CACHE_HOME` moves it). To use another tunnel (ngrok, `tailscale serve`), choose `local` and point the tunnel at port 8787. > [!WARNING] > Anyone with steering rights can make the agent run commands on this machine, as you. That reaches everything you can: your files, Pi Pocket's own settings and sign-in tokens, and its code. A single-session invite limits what someone sees in the app, not what Pi can reach, and Lancet Guard approvals can come from anyone who can steer, including the person who asked. People who can steer every session can also invite others. Invite only people you would trust at your keyboard. View-only users cannot make Pi act. @@ -106,11 +106,21 @@ The flake builds Pi Pocket (`nix run github:TannerMidd/pi-pocket`) and has a Nix | `-y` | | Reuse the last access choice | | `--port` | `8787` | Port to listen on | | `--cwd` | current folder | Default folder for new sessions | -| `--data` | `~/.pi-pocket` | Database, settings, uploads, and push keys | | `--host` | | Listen on a specific address instead of choosing access | | `--rotate-token` | | Issue a new owner link and sign out devices that used the old one | -Environment variables: `PI_POCKET_ACCESS`, `PI_POCKET_DIR`, `PI_POCKET_HOST`, `PI_POCKET_PORT`, `PI_POCKET_GUARD=off`, `PI_POCKET_BROWSER` (the browser to run, when Chromium or Chrome is not found by itself), and `PI_POCKET_BROWSER_ARGS` (extra flags for it, such as `--no-sandbox` where sandboxes are unavailable). Without a terminal (under systemd, for example), the launcher uses `--access` or the last choice. +Pi Pocket keeps its files in the [XDG base directories](https://specifications.freedesktop.org/basedir-spec/latest/), a `pi-pocket` folder in each: + +| Kind | Default | Holds | +| ------ | -------------------------- | ---------------------------------------------------------------------------------------- | +| config | `~/.config/pi-pocket` | People, roles, tokens, and settings; the launcher's last choice | +| data | `~/.local/share/pi-pocket` | The database and its lock, push keys, uploads, worktrees, drop-in extensions, and themes | +| state | `~/.local/state/pi-pocket` | The Cloudflare tunnel's log | +| cache | `~/.cache/pi-pocket` | The downloaded cloudflared, and the built-in browser's profile | + +`XDG_CONFIG_HOME`, `XDG_DATA_HOME`, `XDG_STATE_HOME`, and `XDG_CACHE_HOME` move them, and `PI_POCKET_CONFIG_DIR`, `PI_POCKET_DATA_DIR`, `PI_POCKET_STATE_DIR`, and `PI_POCKET_CACHE_DIR` name each one itself: the `--data` option and `PI_POCKET_DIR` that named one folder are gone. A `~/.pi-pocket` from an older Pi Pocket moves into them on the first start, and is left in place if anything else is in it. + +Environment variables: `PI_POCKET_ACCESS`, `PI_POCKET_HOST`, `PI_POCKET_PORT`, `PI_POCKET_GUARD=off`, `PI_POCKET_BROWSER` (the browser to run, when Chromium or Chrome is not found by itself), and `PI_POCKET_BROWSER_ARGS` (extra flags for it, such as `--no-sandbox` where sandboxes are unavailable), and the four folder variables above. Without a terminal (under systemd, for example), the launcher uses `--access` or the last choice. ## Architecture @@ -152,7 +162,7 @@ Pi knows where Pi Pocket's own documentation is: [docs/index.md](docs/index.md) - Pi Durable is experimental, and its API can change between releases. Versions are pinned in `package.json`. - Only one server process can use the database at a time. - Forks are separate sessions; there is no branch tree to switch between inside one session. -- Pi's own extensions are not supported: Pi Pocket runs its own extensions on Pi Durable, and drop-ins from `~/.pi-pocket/extensions/`. Pi's prompt templates work as slash commands. +- Pi's own extensions are not supported: Pi Pocket runs its own extensions on Pi Durable, and drop-ins from `~/.local/share/pi-pocket/extensions/`. Pi's prompt templates work as slash commands. - A git worktree keeps a session's files apart; it is not a sandbox. Pi can still reach everything you can. - Artifacts run in an opaque-origin sandbox, so `localStorage` and cookies are unavailable inside them. - The browser needs Chromium, Chrome, Brave, or Edge on the server. Its pages live in memory: a restart opens each one again at its last address, but not what was typed into it. Each session's cookies are its own and last until the page closes. A browser installed as a snap (Ubuntu's Chromium) is used only when there is no other; it cannot open files outside your home folder. The list of servers running on the machine is Linux-only. Tested on Linux and macOS; not yet on Windows. diff --git a/SECURITY.md b/SECURITY.md index bd4ab0b3..36eb449d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -16,7 +16,7 @@ Pi Pocket runs a coding agent that executes commands on the machine it runs on, - **Models** are a person's to choose. Pi starts subagents only on their session's model, or on models the owner allowed (Providers → Models for subagents); PiP passes a model only when its person named it; titles and summaries use the session's own model. Subagents on another model, and PiP, ask a model's sign-in afresh first and use none whose sign-in expired; a session's own run, a subagent on its model, titles, and summaries just try the model, and a run that fails on an expired sign-in marks it. This guards spending against Pi's choices, not against a person: anyone who can steer can pick a session's model, and Pi, which runs commands, can reach Pi's sign-ins and Pi Pocket's files like anything else on the machine. - **Linked sessions** (Agent Teams, off until the owner turns it on) widen what prompt injection can reach. A session that read something hostile (a page, a file, an issue) can pass it on to the other sessions of its team in a message, and a message makes Pi work in the target for the person Pi worked for in the sender, with that person's access, in a session where other people may have written. Pi is told to take a linked session's message as a request, not as an order from a person, and to do nothing destructive or with credentials only because one asks; that is guidance to the model, not a boundary. A message is fenced, and a team stops after 20 messages without a person, at the starting session's spend limit, and at 10 sessions; a link gives no access to a session someone cannot already see. - **Plan mode** keeps Pi to planning; it is not a boundary. Programs that people configured to run on their own, such as a git diff driver, run as they would for anyone. -- **Drop-in extensions** in `~/.pi-pocket/extensions/` run inside the server with the owner's rights. They stay off until the owner turns them on. One with a browser module also runs code in the app's own page, with the rights of whoever's tab loads it: the owner's alone unless the module opens it to others (never to someone invited to one session). The page's content security policy keeps that code from loading scripts and styles from another site or fetching from one, but code in the page could still send data out in other ways (navigating the page elsewhere, for one): turn on only browser modules you trust, as with the server part. What a module's route throws with `web.error` reaches everyone who may use that route as it is; anything else it throws, only the owner reads. Every file of a web type (`.js`, `.mjs`, `.css`, `.json`, images, and `.woff2` fonts) in a browser module's folder can be read by everyone the module is for. A module's conversation command is offered to everyone who can steer that session, people invited to only that session included, and its handler runs with the owner's rights: treat its arguments as untrusted, and check who runs it. A global command is the owner's alone. +- **Drop-in extensions** in `~/.local/share/pi-pocket/extensions/` (`XDG_DATA_HOME` moves it) run inside the server with the owner's rights. They stay off until the owner turns them on. One with a browser module also runs code in the app's own page, with the rights of whoever's tab loads it: the owner's alone unless the module opens it to others (never to someone invited to one session). The page's content security policy keeps that code from loading scripts and styles from another site or fetching from one, but code in the page could still send data out in other ways (navigating the page elsewhere, for one): turn on only browser modules you trust, as with the server part. What a module's route throws with `web.error` reaches everyone who may use that route as it is; anything else it throws, only the owner reads. Every file of a web type (`.js`, `.mjs`, `.css`, `.json`, images, and `.woff2` fonts) in a browser module's folder can be read by everyone the module is for. A module's conversation command is offered to everyone who can steer that session, people invited to only that session included, and its handler runs with the owner's rights: treat its arguments as untrusted, and check who runs it. A global command is the owner's alone. - **The extension gallery** (Menu → Extensions → Find extensions) installs drop-ins other people wrote, so what it installs has the owner's rights once turned on. It reads its index strictly, downloads only over https from each entry's pinned commit, refuses a file whose SHA-256 differs from the entry's, keeps file names inside the drop-in folder, and installs everything off; updates are never automatic. The index's checks and its maintainers' review lower the risk, but do not remove it: read the code (or ask Pi to review it) before you turn one on. Only the owner can use the gallery. A theme from the gallery is CSS, fonts, and images that the owner chooses to install. Pi Pocket checks it as every theme folder before it is in place, so it cannot load anything from other servers, nor the app's own addresses, and it runs no code. It can still hide or relabel parts of the app, approvals too: install only themes you trust. - **Replies are untrusted.** Anything Pi reads can steer what it writes, so the app page loads nothing from other sites, and replies cannot contain forms, scripts, or styling. Artifacts run in a sandbox with no access to the app. An HTML block in a reply is previewed in a frame of its own with its scripts off and a policy that loads nothing, not even from the app; tapping **Run** runs it as an artifact runs, with scripts and the network but no access to the app, so a page Pi was steered into writing could then send what it shows elsewhere. SVG blocks show as images, which run nothing. diff --git a/docs/architecture.md b/docs/architecture.md index 0fc97876..f524d321 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -4,7 +4,7 @@ How Pi Pocket's parts depend on each other, and the orderings the code relies on ## Durable truth and what memory keeps -Pi Durable owns the lasting state of conversations: conversations, entries, tasks, and Pi Pocket's documents (`docs.ts`), in `pocket.sqlite`. People and settings, push subscriptions, uploads, and worktrees are files beside it ([map.md](map.md#data-pi-pocket-or-pi_pocket_dir)). What `PocketApp` keeps in memory about conversations is read back from storage: +Pi Durable owns the lasting state of conversations: conversations, entries, tasks, and Pi Pocket's documents (`docs.ts`), in `pocket.sqlite`. People and settings, push subscriptions, uploads, and worktrees are files in Pi Pocket's folders ([map.md](map.md#data-the-xdg-folders-or-pi_pocket_dir)). What `PocketApp` keeps in memory about conversations is read back from storage: - At startup, `#recover` reads each conversation's run, agent, chat, subagents, and who Pi works for in it. - After that, every commit goes through `#committed`, which keeps the memory up to date. diff --git a/docs/design/theming.md b/docs/design/theming.md index a57bd476..287cd900 100644 --- a/docs/design/theming.md +++ b/docs/design/theming.md @@ -28,7 +28,7 @@ The goal, from issue #90, point 3: "A theme should be a folder. Copy the main th ## The decisions -1. **A theme is a folder:** `~/.pi-pocket/themes//`. The built-in **Pocket** theme is `web/themes/pocket/`, in the same format. +1. **A theme is a folder:** `~/.local/share/pi-pocket/themes//` (`XDG_DATA_HOME` moves it). The built-in **Pocket** theme is `web/themes/pocket/`, in the same format. 2. **Two choices: Theme and Colors.** The theme sets the look and the layout. Colors is a palette: the 22 Omarchy palettes, Desktop, or a folder's `colors.toml`. A layout must stay when the colors change, because Follow desktop and the launcher's previews change colors often. With one list, a layout would need a copy for each palette. 3. **A folder holds colors, CSS, or both.** `colors.toml` is Omarchy's format, unchanged, so any Omarchy theme folder dropped in shows under Colors. When you choose a theme that has its own colors, Colors changes to them too. You can change it after. 4. **Duplicate gives a starter, not a fork.** If Duplicate copied all of `style.css`, you would have a 14,700-line fork that falls behind at each release. So Pocket's folder holds a manifest and a `theme.css` that lists every token, commented out with its default, plus examples for each part. @@ -46,7 +46,7 @@ The goal, from issue #90, point 3: "A theme should be a folder. Copy the main th ## The format by example ``` -~/.pi-pocket/themes/paper/ +~/.local/share/pi-pocket/themes/paper/ theme.json theme.css layout.css (theme.css imports it) diff --git a/docs/examples/clock.ts b/docs/examples/clock.ts index bb69529d..afec8561 100644 --- a/docs/examples/clock.ts +++ b/docs/examples/clock.ts @@ -1,6 +1,6 @@ /** * A clock tool, and a /clock-zone command that sets the time zone it uses in a session. An example drop-in extension: - * copy it into `~/.pi-pocket/extensions/`, then turn it on in Menu → Extensions. + * copy it into `~/.local/share/pi-pocket/extensions/`, then turn it on in Menu → Extensions. */ import { Type } from "@earendil-works/pi-ai"; import { defineDoc, defineExtension, defineTool } from "@earendil-works/pi-durable"; diff --git a/docs/examples/no-force-push.ts b/docs/examples/no-force-push.ts index d98aa279..d73291b2 100644 --- a/docs/examples/no-force-push.ts +++ b/docs/examples/no-force-push.ts @@ -1,6 +1,6 @@ /** * No force-pushing: a bash call that force-pushes with git is blocked before it runs, and the system prompt says so. - * An example drop-in extension with a hook and a prompt section: copy it into `~/.pi-pocket/extensions/`, then turn it + * An example drop-in extension with a hook and a prompt section: copy it into `~/.local/share/pi-pocket/extensions/`, then turn it * on in Menu → Extensions. */ import { defineExtension, hook, section, ToolTask } from "@earendil-works/pi-durable"; diff --git a/docs/examples/server-status.ts b/docs/examples/server-status.ts index bdf6d819..88c6bbf4 100644 --- a/docs/examples/server-status.ts +++ b/docs/examples/server-status.ts @@ -1,6 +1,6 @@ /** * Server status: a page on the rail with this machine's load, memory, and uptime, which stays live while it shows. - * An example drop-in extension with a web part: copy it and `server-status.js` into `~/.pi-pocket/extensions/`, then + * An example drop-in extension with a web part: copy it and `server-status.js` into `~/.local/share/pi-pocket/extensions/`, then * turn it on in Menu → Extensions. */ import { cpus, freemem, hostname, loadavg, totalmem, uptime } from "node:os"; diff --git a/docs/extensions.md b/docs/extensions.md index 7f96601a..95e6a349 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -40,7 +40,7 @@ This page is the map. Read all of it, then only the pages that your task needs. ## Drop-in or built-in -**A drop-in** is a file in the drop-in folder, `~/.pi-pocket/extensions/` (the `extensions/` folder in the data folder). Use a drop-in for all that one person or one machine wants. +**A drop-in** is a file in the drop-in folder, `~/.local/share/pi-pocket/extensions/` (the `extensions/` folder in the data folder; `XDG_DATA_HOME` moves it). Use a drop-in for all that one person or one machine wants. - It stays when Pi Pocket updates. - It starts off. The owner turns it on in Menu → Extensions. diff --git a/docs/extensions/first-extension.md b/docs/extensions/first-extension.md index 838ec0b1..51dab669 100644 --- a/docs/extensions/first-extension.md +++ b/docs/extensions/first-extension.md @@ -8,7 +8,7 @@ In this tutorial, you write a drop-in extension. It gives Pi a tool that counts - You must have Pi Pocket 0.14.0 or later. - You must be able to write a file on the machine where Pi Pocket runs. You can also ask Pi to write the file for you. -Drop-ins go in the drop-in folder: `~/.pi-pocket/extensions/`, or `extensions/` in `PI_POCKET_DIR`. Menu → **Extensions** shows the path at the bottom of the sheet. +Drop-ins go in the drop-in folder: `~/.local/share/pi-pocket/extensions/` (`XDG_DATA_HOME` moves it), or `extensions/` in `PI_POCKET_DATA_DIR`. Menu → **Extensions** shows the path at the bottom of the sheet. ## 1. Write the module diff --git a/docs/extensions/policies/api-stability.md b/docs/extensions/policies/api-stability.md index c1ca9b7f..17be01ed 100644 --- a/docs/extensions/policies/api-stability.md +++ b/docs/extensions/policies/api-stability.md @@ -106,7 +106,7 @@ The host's members do not change while modules reload. A new member comes only a - The app's HTTP API, other than a module's own routes under `/api/x//`. - The events between the server and its tabs, and the `pocket.*` keys in browser storage. - Pi Pocket's document kinds (`pocket.*`), and the shapes of its documents, entries, and tasks. A module can read them with `api.snapshot`, but their fields can change. -- `config.json`, `pocket.sqlite`, and the rest of the data folder, except `extensions/`. +- `config.json`, `pocket.sqlite`, and the rest of Pi Pocket's folders, except `extensions/`. - All CSS that is not in [CSS](#7-css). - The words of errors, notices, logs, and the system prompt. - The tools of built-in extensions: their arguments and results. Only their names are public, as reserved names. diff --git a/docs/extensions/policies/core-or-extension.md b/docs/extensions/policies/core-or-extension.md index 608a3ab4..5a152deb 100644 --- a/docs/extensions/policies/core-or-extension.md +++ b/docs/extensions/policies/core-or-extension.md @@ -8,7 +8,7 @@ This page tells where a change goes: into Pi Pocket's core, or into an extension - An **extension** is a module that Pi Pocket loads through its extension loader. The owner can turn each one off in **Menu → Extensions**, except the system prompt (`prompt.ts`). There are three types: - A **built-in extension** is a module in `src/server/extensions/`. It comes with Pi Pocket. - A **gallery extension** is a drop-in that its author keeps in their own repository. The gallery lists it ([gallery-review.md](gallery-review.md)). - - A **drop-in** is a module in the owner's `~/.pi-pocket/extensions/` folder. + - A **drop-in** is a module in the owner's `~/.local/share/pi-pocket/extensions/` folder. - An **extension point** is a part of the public API that lets an extension add to Pi Pocket or change it. [api-stability.md](api-stability.md) lists them. ## The decision @@ -85,7 +85,7 @@ Ask these questions in this order. The first "yes" gives the answer. ### Themes (#90, 0.14.0) - **Core:** the theme surface, which every theme can change: tokens (CSS variables), the `data-part` names of the parts of the screen, and their documented states. Also the loading and the checks of theme folders. -- **Not core:** each theme, as a folder in `~/.pi-pocket/themes/`. Duplicate the Pocket theme, change the copy, and choose it in settings. +- **Not core:** each theme, as a folder in `~/.local/share/pi-pocket/themes/`. Duplicate the Pocket theme, change the copy, and choose it in settings. - **Why:** the tokens and the parts must be the same for every theme (C5). The look is a preference (C6). The default theme stays the one that Pi Pocket ships. - The theme guide and the theme surface reference tell what a theme can change. diff --git a/docs/extensions/reference/glossary.md b/docs/extensions/reference/glossary.md index 284ccf5f..18dfe498 100644 --- a/docs/extensions/reference/glossary.md +++ b/docs/extensions/reference/glossary.md @@ -102,7 +102,7 @@ A module in `src/server/extensions/`. It is part of Pi Pocket. ### data folder -Pi Pocket's folder for its data: `~/.pi-pocket` by default. Code: `host.dataDir`. +Pi Pocket's folder for its data: `~/.local/share/pi-pocket` by default (`XDG_DATA_HOME` moves it). Code: `host.dataDir`. ### drop-in folder diff --git a/docs/extensions/reference/host.md b/docs/extensions/reference/host.md index c654f748..be33d20c 100644 --- a/docs/extensions/reference/host.md +++ b/docs/extensions/reference/host.md @@ -28,7 +28,7 @@ readonly dataDir: string; Since 0.1.0 · Stable -Pi Pocket's data folder. It is `~/.pi-pocket`, unless `PI_POCKET_DIR` or the `--data` option gives another folder. The drop-in folder is `extensions/` in it. +Pi Pocket's data folder. It is `~/.local/share/pi-pocket`, unless `XDG_DATA_HOME` or `PI_POCKET_DATA_DIR` gives another folder. The drop-in folder is `extensions/` in it. ### `agentDir` diff --git a/docs/extensions/themes.md b/docs/extensions/themes.md index ca50f805..ea80a5e4 100644 --- a/docs/extensions/themes.md +++ b/docs/extensions/themes.md @@ -25,14 +25,14 @@ Besides Appearance: the launcher's **Themes** (Ctrl/⌘+K, then `#`) lists the t Each save shows at once in every tab that uses the theme. The page does not reload, so nothing you typed is lost. -You can also edit the folder yourself: `~/.pi-pocket/themes//` (or `themes/` in `PI_POCKET_DIR`). It is the one place in the data folder, with `extensions/`, that you edit by hand. **Remove** moves a theme's folder to `themes/.removed/`, so you can get it back from there. Pi Pocket never deletes what is in `.removed/`, so it grows: delete the folders there you no longer want. +You can also edit the folder yourself: `~/.local/share/pi-pocket/themes//` (or `themes/` in `PI_POCKET_DATA_DIR`; `XDG_DATA_HOME` moves it). It is the one place in the data folder, with `extensions/`, that you edit by hand. **Remove** moves a theme's folder to `themes/.removed/`, so you can get it back from there. Pi Pocket never deletes what is in `.removed/`, so it grows: delete the folders there you no longer want. A copy of Pocket starts with a `theme.css` that lists every token with its value, commented out. Take the comment marks off a token and change its value. Leave the others commented out, so they follow the app's updates. ## The folder ``` -~/.pi-pocket/themes/paper/ +~/.local/share/pi-pocket/themes/paper/ theme.json theme.css layout.css (theme.css imports it) diff --git a/docs/features.md b/docs/features.md index 6b13ac9b..cc7c14ba 100644 --- a/docs/features.md +++ b/docs/features.md @@ -45,7 +45,7 @@ The full tour. For setup, see the [README](../README.md). - **Worktrees.** A new session (the New session sheet) or a fork (the message sheet) can work in a git worktree of its own: a branch, apart from the folder it came from, which starts with that folder's files as they are. Menu → **Worktree** removes it (not while Pi works in it); its branch stays. - **Continue a Pi session.** A session of Pi in the terminal can carry on here. The owner picks it from **Continue a Pi session** in the New session sheet (or the launcher): Pi's sessions on this computer by folder, newest first, with a search through their titles, folders, and words, and for one its folder, model, and last messages. **Continue in Pocket** makes a new session in its folder, with its model when that one is signed in here, and its whole conversation to read. Pi gets the context Pi itself builds from the file: what a compaction summarized, and Pi's context edits, stay as Pi had them. Pi's system prompt and extensions' state stay behind, and nothing Pi did runs again: a tool call that never finished gets a result saying so. The session's file is only read. The list marks sessions already continued here, and says when Pi went on in the terminal since. - **Export.** Menu → **Export as Markdown** (or `/export`): the whole history as a Markdown file, with tool calls folded into `
` blocks. -- **Any provider or model.** Uses Pi's own model runtime and sign-ins (`~/.pi/agent/auth.json`): pick a model and thinking level per session from the menu that opens up from the model chip in the message box (arrows and Enter pick, with a search box when there are many models), and sign in to providers (API key or subscription) from an inline card in the Providers sheet; expired sign-ins come first with a warning pill. API keys cannot be empty or only spaces. A failed sign-in shows a short message, technical details folded away, and **Start again**. A provider search with no matches offers **Clear search**. Pi’s `enabledModels` setting supplies the **Scoped** list, using Pi’s own pattern matching (including wildcards and thinking-level suffixes). When any scoped models are available, **[scoped | all]** beside the **Model** heading switches between that list and every model with a configured sign-in; the active choice uses the primary color. Tab in the search box (or in the picker, when it has no search box) switches it too, also when the search has no matches; from the picker's other controls, Tab moves the focus. On a touch screen, each choice takes a tap about 40px across. Each opening starts scoped; without a scope, when no pattern matches, or when `/model …` searches for models only outside the scope, it shows all instead. Switching the list keeps the search and changes no model or setting. The default-model picker has the same switch. New sessions start with the owner's default model and thinking level: **Make default** under the thinking levels makes the session's the default, and Menu → **Default model** (or the launcher) picks one without changing a session; without one, a new session starts with a trusted project's model/thinking default, then the last model anyone picked, then Pi's global default. To use project defaults, Menu → **Default model** → **Clear owner default: use project defaults, then last picked**. A project pin needs both `defaultProvider` and `defaultModel` in the final session folder's `.pi/settings.json`; `defaultThinkingLevel` is optional (`off` when omitted) and limited to that model's levels. Thinking-only project settings do not change another source's pair. Untrusted settings use normal fallback; a settings-only project appears in the owner's trust flow. Trust affects only later new sessions, never the open one, and no project setting can grant itself trust. Invalid consulted trusted settings stop creation with a contained error, without a model swap. Known unsigned project models stay selected. Imports, forks, subagents, manual choices, and PiP's own conversation keep their previous behavior. Extra per-project skill/prompt paths and per-project run policy are not supported yet ([project defaults](customizing.md#project-model-defaults-for-new-sessions)) A model is always one a person chose: a default that is not signed in now stays the default (the chip shows it, and the first message says why it fails) rather than give way to another model, and with no model chosen anywhere a new session starts with none until someone picks one. A subscription sign-in that its provider will no longer renew (OpenAI's refusing a refresh with `invalid_grant`, say) is found when the owner opens the Providers sheet, when a subagent on another model or PiP is about to use one of its models, or when a run fails on it: the sheet marks the provider **needs sign-in** with what the provider said, the model menu marks its models, the owner (alone) gets a notice, and subagents on other models and PiP use none of its models until someone signs in again. A session on one of its models, and a subagent on that session's model, are not checked first: they try it, and their runs fail where you see them. **Models for subagents**, at the top of the owner's Providers sheet, lists every signed-in model by provider, with a box to find one when there are many: each one checked is a model Pi may start subagents on besides their session's own, none by default (see Background subagents; `subagentModels` in `~/.pi-pocket/config.json`). A tap changes that one model alone, so quick taps, and two tabs at once, keep every choice. An allowed model that is not signed in now stays listed, marked, so it can be taken off. +- **Any provider or model.** Uses Pi's own model runtime and sign-ins (`~/.pi/agent/auth.json`): pick a model and thinking level per session from the menu that opens up from the model chip in the message box (arrows and Enter pick, with a search box when there are many models), and sign in to providers (API key or subscription) from an inline card in the Providers sheet; expired sign-ins come first with a warning pill. API keys cannot be empty or only spaces. A failed sign-in shows a short message, technical details folded away, and **Start again**. A provider search with no matches offers **Clear search**. Pi’s `enabledModels` setting supplies the **Scoped** list, using Pi’s own pattern matching (including wildcards and thinking-level suffixes). When any scoped models are available, **[scoped | all]** beside the **Model** heading switches between that list and every model with a configured sign-in; the active choice uses the primary color. Tab in the search box (or in the picker, when it has no search box) switches it too, also when the search has no matches; from the picker's other controls, Tab moves the focus. On a touch screen, each choice takes a tap about 40px across. Each opening starts scoped; without a scope, when no pattern matches, or when `/model …` searches for models only outside the scope, it shows all instead. Switching the list keeps the search and changes no model or setting. The default-model picker has the same switch. New sessions start with the owner's default model and thinking level: **Make default** under the thinking levels makes the session's the default, and Menu → **Default model** (or the launcher) picks one without changing a session; without one, a new session starts with a trusted project's model/thinking default, then the last model anyone picked, then Pi's global default. To use project defaults, Menu → **Default model** → **Clear owner default: use project defaults, then last picked**. A project pin needs both `defaultProvider` and `defaultModel` in the final session folder's `.pi/settings.json`; `defaultThinkingLevel` is optional (`off` when omitted) and limited to that model's levels. Thinking-only project settings do not change another source's pair. Untrusted settings use normal fallback; a settings-only project appears in the owner's trust flow. Trust affects only later new sessions, never the open one, and no project setting can grant itself trust. Invalid consulted trusted settings stop creation with a contained error, without a model swap. Known unsigned project models stay selected. Imports, forks, subagents, manual choices, and PiP's own conversation keep their previous behavior. Extra per-project skill/prompt paths and per-project run policy are not supported yet ([project defaults](customizing.md#project-model-defaults-for-new-sessions)) A model is always one a person chose: a default that is not signed in now stays the default (the chip shows it, and the first message says why it fails) rather than give way to another model, and with no model chosen anywhere a new session starts with none until someone picks one. A subscription sign-in that its provider will no longer renew (OpenAI's refusing a refresh with `invalid_grant`, say) is found when the owner opens the Providers sheet, when a subagent on another model or PiP is about to use one of its models, or when a run fails on it: the sheet marks the provider **needs sign-in** with what the provider said, the model menu marks its models, the owner (alone) gets a notice, and subagents on other models and PiP use none of its models until someone signs in again. A session on one of its models, and a subagent on that session's model, are not checked first: they try it, and their runs fail where you see them. **Models for subagents**, at the top of the owner's Providers sheet, lists every signed-in model by provider, with a box to find one when there are many: each one checked is a model Pi may start subagents on besides their session's own, none by default (see Background subagents; `subagentModels` in `~/.config/pi-pocket/config.json`). A tap changes that one model alone, so quick taps, and two tabs at once, keep every choice. An allowed model that is not signed in now stays listed, marked, so it can be taken off. - **Multiplayer.** Any number of browsers can watch and steer the same session live. The status line under the message box shows how many are here, how full the context is (after a compaction, an estimate marked ~ until the next answer measures it), the cache hit rate, the cost, and a pill that says whether Lancet Guard is on (it opens Extensions). **Reply speed** in Appearance (or the launcher) adds how fast the last answer came: its tokens per second and how long its first token took (a response delivered in one buffered flush has no speed, only that wait). It is off until you turn it on, in each browser. On a touch screen, a tap on the line says in a notice what its items mean. On a narrow screen the line keeps to one row where it can. Where it does not fit, it gives up these in turn, only as far as it must: the speed's long words ("42 tok/s · 1.2s"), "1 here" while you are alone, the word "guard" (the shield stays), the word "context" (the meter stays), the cache, and the wait for the first token. "Sign in another device" makes a one-time invite link, a QR code, and a short code to type on the other device's sign-in screen. The owner can tick **Sign in as me (owner)** to sign their own other device in as the owner, with the owner's notifications, instead of as someone new. Messages show who sent them. - **Working together.** The top bar shows who else is in the session (an avatar with a mark that greys when their tab is in the background, and how many more), and "Alex is writing to Pi…" warns before two people steer at once. The **People** panel opens beside the conversation on wide screens, where it takes the Browser panel's place, and as a sheet on phones. Its header counts the people here; the roster shows their online or away state, typing, and role. A **Take turns** card starts or turns off shared driving with the same permissions as the menu. It has: - **Chat** for the people in the session, which Pi does not see. Your messages are right-aligned accent bubbles; other people's are left-aligned, with their name and time above. Quotes sit above the bubble, and the pill-shaped message box keeps a dismissible quote strip. The latest pin stays above the chat. Type `@` to mention someone (they get a notice wherever they are), tap **Discuss** under a reply of Pi's to quote it, and tap a chat message to **Send to Pi**, pin, or copy it. Changes show up here as activity lines ("Alex stopped the run", "Tanner switched the model to …", "Alex allowed the bash call: …"), and as a notice to everyone else in the session. @@ -54,12 +54,12 @@ The full tour. For setup, see the [README](../README.md). - Under Pi's answers: emoji reactions, **Discuss**, and **Pin**. Queued messages show who queued them. Approved or denied tool calls show who decided. - **Take turns.** Menu → **Take turns**: one person drives and only they send to Pi or change its settings. Others tap **Ask to drive**, the driver hands over, and anyone can take the wheel when the driver has left (the owner always can). Anyone who can steer can still stop a run. - **Roles and invites.** An invite can let someone **steer** or make them **view only** (they read, react, and chat, but cannot send to Pi, approve, or upload). It can cover **every session** or **only this session**, and lasts 15 minutes, an hour, a day, or a week (**Expires after**); either way it works once, and a restart keeps it. Changing what the sheet's invite grants, or how long it lasts, ends the one it showed; **New invite** makes another and leaves the one before working. Only the owner can make an owner invite (**Sign in as me (owner)**): it makes no one new, but signs the device in with the owner's token, so it has full control, works once, and always expires in 15 minutes; rotating the owner token (`--rotate-token`) signs out every device signed in that way, and ends owner invites not yet used. Menu → **People** lists everyone, who is online, and when the others were last here. The owner can change what each person may do or remove them. The session list shows who is in each session, and a dot marks new chat messages. -- **Notifications.** Menu → **Notifications** turns on push notifications for a device: Pi finished, Pi needs approval, someone @mentioned you, or a chat message in a session you take part in. You only get them while you are not looking at that session. An approval can be denied from the notification, and allowed there when the whole call fits on one short line. The home-screen icon counts the sessions that need you. They need an https address: the Cloudflare Tunnel, or `tailscale serve` in front of **This device only** (`--access local`). On iPhone, add Pi Pocket to the home screen first. The server signs pushes with its own keys in `~/.pi-pocket/push.json`. +- **Notifications.** Menu → **Notifications** turns on push notifications for a device: Pi finished, Pi needs approval, someone @mentioned you, or a chat message in a session you take part in. You only get them while you are not looking at that session. An approval can be denied from the notification, and allowed there when the whole call fits on one short line. The home-screen icon counts the sessions that need you. They need an https address: the Cloudflare Tunnel, or `tailscale serve` in front of **This device only** (`--access local`). On iPhone, add Pi Pocket to the home screen first. The server signs pushes with its own keys in `~/.local/share/pi-pocket/push.json`. - **Built-in browser.** Menu → **Browser** (or Alt+B, `/browser localhost:5173`, or the globe in the top bar while a page is open; on a phone, a dot on the places button by the message box says so) opens a real Chromium page that runs on the server: beside the conversation on wide screens, full screen on phones. Type an address (`localhost:5173`, `example.com`), or (the owner) tap a row under **Running on this machine** on the start screen; a green dot in the rounded address field marks an open page. Tap, drag to scroll, and type into the page (the keyboard button sends text from a phone). The size button switches between fitting the panel (the page follows the panel when it changes size), desktop, phone, and tablet sizes, and the console button shows the page's logs and errors, with a warning-colored error count. Pi's `browser` tool uses the same page: it opens addresses (dev servers, HTML files on the machine, any site), reads a snapshot of the page's text and controls, clicks, types, takes screenshots it can see, reads the console, and runs JavaScript in the page, while everyone in the session watches. Each session has one page, in a browser context of its own; links that open a new tab open there instead, and dialogs are accepted. Anyone in the session can watch; using the page takes steering rights (and the wheel, while take turns is on), the addresses people open show in the chat, and only the owner opens files from the address bar. The browser reaches what the server reaches, `localhost` and its network included, and Lancet Guard does not check it (see SECURITY.md). A restart closes the pages, and each opens again at its last address and size. Where the panel covers the conversation, Esc closes it, unless the page has the keys (then Alt+B does). In plan mode Pi may look at pages but not click, type, or run scripts in them. It needs Chromium, Chrome, Brave, or Edge on the server (or `PI_POCKET_BROWSER`); the owner can turn it off in Extensions, which closes the panel too. - **Artifacts.** The `artifact` tool publishes versioned HTML pages, Markdown documents (shown in the app's theme), or SVG images. Give it `content`, `edits` against the latest version, or a `path` to a UTF-8 text file on the server (up to 2 MB). A path can be absolute, start with `~/`, or be relative to the session's folder. The file is copied into a durable version, not linked: changing or deleting the source does not change the artifact. For a new artifact, `.html`, `.md`/`.markdown`, and `.svg` infer its type; a path update keeps the existing id's type. Images and neighboring assets are not bundled. The Artifacts sheet lists every version in a rounded card grid; opening one shows the sandboxed viewer (or its own tab). The list does not run artifacts as previews. Artifacts cannot reach the app or its cookies. - **Agent Teams.** Off until the owner turns it on in Menu → **Extensions**. Pi starts work in a linked session: an ordinary session in the list, under the one that started it, with its own settings and life (stopping or archiving one leaves the other as it is). It works in the same folder (no worktree of its own) with the same model, thinking level, and agent settings, but none of the history, plan mode, or budget, so Pi gives it all it needs in its first message. Pi may name another model only from the owner's **Models for subagents** (Menu → Providers), and a lower thinking level, never a higher one; a model that cannot be used now is refused, with no session made. Pi's tool, `team`, starts linked sessions, lists them, and sends them messages, both ways: an answer does not go back by itself, so each one sends back what it was asked for. A message from a linked session shows as "via a linked session", and Pi takes it as a request, not as a person's order. The links above the transcript (**Started from**, **Linked session**, and past three, **+N more**) open the others. Each start and message acts as the person Pi works for, with their access, take turns, plan mode, and spend limit: a session someone cannot see stays out of their reach, and its name out of the list Pi gets, unless everyone in the session sees it. A link gives no access: someone who sees only the new session does not learn where it came from. A team stays small: a session starts at most 10 linked sessions, a linked session starts none, the budget of the session that started the team covers what all of them spend, and after 20 messages between them with no person writing, they stop until a person writes in one. Unlike subagents, linked sessions report nothing by themselves, and a subagent cannot start them. - **Background subagents.** The `subagent` tool spawns named subagents that work in their own conversations while you keep talking. Open one to watch it or talk to it. Its answer goes back to the main agent at its next pause (after a tool call; when it is idle, after a moment, so answers that finish together go together); answers that arrive meanwhile wait together and go as one message, shown as a card each, so many subagents cost the main agent one extra turn at most, and wait in its queue as one row: one that arrives while that row waits joins it. A subagent that fails says why, and one that is stopped says so, to the main agent too, so it does not wait for an answer; a message sent to it while it worked, which its failed run never took, is withdrawn, and the main agent is told to send it again. A subagent runs on its session's model, thinking as hard as the session or less (a level its model lacks becomes the nearest lower one it has, and the main agent is told), with an optional tool list. Pi picks no model: until the owner allows other models for subagents (**Models for subagents** in the Providers sheet), the subagent tool offers none and refuses one Pi names; then Pi may use only those, and one whose sign-in expired is refused too, with word to tell you rather than pick another. A subagent on a model the owner no longer allows gets no more work, and one thinking harder than its session (started before this, or raised by hand) is lowered to the session's level as work is sent. One whose model or provider failed (an expired sign-in, a model that is gone or refused, a quota or rate limit) says so with its model, on its card and in a notice, and the main agent is told to stop and tell you, never to start it again on another model. -- **PiP.** The owner's assistant, "Pi in Pocket", over every screen. It stays out of the way until called: Alt+A, the rail's orb, the PiP tile in the places, the launcher, or `/pip` (`/pip what needs me?` asks at once); a small mark shows in the corner only while it works or has something new. It opens as a rounded, shadowed window anchored 16px from the bottom right corner on wide screens (on a short screen it is shorter, never over a session's top bar; on a screen narrower than about 1600 px it covers the right end of a session's message box until it closes) and a sheet from the bottom on phones, over whatever is open, which stays as it was; Esc, back, or a swipe down closes it. PiP looks after the other sessions: it lists and reads them, starts sessions (in a folder, a worktree, with a model) and messages them, stops and archives them, schedules messages there, watches your GitHub repositories and pull requests and acts on what happens there (see Watch a repository), starts sessions in a repository by name (its checkout, or a clone), and follows runs until they end, when a report comes back to it and it says in a line what came of it: pushed as a notification when you are not looking (instead of the session's own "Pi finished", which is held back only while PiP is on, not held back by a spend limit, and following that run, and for a minute after its report), which opens PiP over whatever the app shows. A request to do hands-on work authorizes PiP to start or message the session it needs; you do not have to say “start a session.” PiP delegates to that session's tools instead of refusing because its own tools cannot edit files or run commands. PiP's model and new sessions' default model are separate: if a worker has no model, choose one in that session. Menu → Default model sets the model for future sessions. It is dependable before fast: it acts on a session (messages, stops, archives, follows its run, schedules there, or watches its PR) only after it looked at it in the last 15 minutes (read it, started it, or had a report from it), its tools say how a session stands after they act, and it says what it checked and what it did not; tasks it sends that touch code ask for the checks and tests. It never picks a model: which one is yours to say, as below. What sessions say reaches PiP fenced as data, never as instructions, and what sessions, guests, or GitHub named (titles, waiting calls, scheduled messages, PR titles) comes on one line, quoted, as data too. It makes at most 60 messages, follows, schedules, and watches (its reminders to itself included; a schedule it could not make does not count), and starts at most 20 sessions, an hour (counted since the server started). It can deny a call waiting for approval, never allow one. When one of its own calls waits for approval, the panel says so, and **Review** opens its conversation, where it can be allowed. What it sends shows as sent through PiP for you; it reads files but changes none and runs no commands, handing hands-on work to sessions. Your messages are accent bubbles; completed steps have small green checks, and reports are rounded cards that expand to show more. Its replies link the sessions they name; **Open as a conversation** shows every step, and **Start fresh** gives it a new context. Its context stays small, as PiP needs no long memory of its own: the panel's header measures context against PiP's compaction limit, capped at the model's context window (or just that window with **As sessions do**), and follows changes in Extensions; **Compact** there (or `/compact`, or `/compact` with what to keep in mind, typed to PiP) turns its older messages into a short summary, the newest few kept as they are; and it compacts itself once idle above 60k tokens of context (the owner picks 40k, 60k, 100k, or only as sessions do, in Extensions under PiP; `pipCompactAt` in `~/.pi-pocket/config.json`), then waits for 8k tokens more before it does again. A session's thresholds (Pi's own settings) stay as a backstop during a long run. PiP's summary is its own: written from its conversation with what sessions said, webhooks' posts, GitHub's events, and what files and scripts gave back left out, so none of their words come back as PiP's notes; it keeps its person's preferences, what is still open, and what it follows and watches, which live on the server and carry on as they were. A summary that cannot be written leaves the context as it was: Compact's notice says why, and after one of PiP's own that failed it waits 30 minutes, or 8k tokens more, before it tries again. A summary costs a request, counted in PiP's spend. It is a conversation of its own, kept out of the session list and out of everyone else's reach (nobody else can open it or write to it), and durable like the rest: follows outlive a restart. Its cost shows in its person's Spend sheet, and its follows in Running now. Turn it off in Extensions: it stops what it was doing (a compaction too), takes no messages, its own conversation's box included, and meanwhile its follows' reports and its own scheduled messages wait, webhooks to it answer 503, and repository watches to it hold what they found, all going on once it is back. +- **PiP.** The owner's assistant, "Pi in Pocket", over every screen. It stays out of the way until called: Alt+A, the rail's orb, the PiP tile in the places, the launcher, or `/pip` (`/pip what needs me?` asks at once); a small mark shows in the corner only while it works or has something new. It opens as a rounded, shadowed window anchored 16px from the bottom right corner on wide screens (on a short screen it is shorter, never over a session's top bar; on a screen narrower than about 1600 px it covers the right end of a session's message box until it closes) and a sheet from the bottom on phones, over whatever is open, which stays as it was; Esc, back, or a swipe down closes it. PiP looks after the other sessions: it lists and reads them, starts sessions (in a folder, a worktree, with a model) and messages them, stops and archives them, schedules messages there, watches your GitHub repositories and pull requests and acts on what happens there (see Watch a repository), starts sessions in a repository by name (its checkout, or a clone), and follows runs until they end, when a report comes back to it and it says in a line what came of it: pushed as a notification when you are not looking (instead of the session's own "Pi finished", which is held back only while PiP is on, not held back by a spend limit, and following that run, and for a minute after its report), which opens PiP over whatever the app shows. A request to do hands-on work authorizes PiP to start or message the session it needs; you do not have to say “start a session.” PiP delegates to that session's tools instead of refusing because its own tools cannot edit files or run commands. PiP's model and new sessions' default model are separate: if a worker has no model, choose one in that session. Menu → Default model sets the model for future sessions. It is dependable before fast: it acts on a session (messages, stops, archives, follows its run, schedules there, or watches its PR) only after it looked at it in the last 15 minutes (read it, started it, or had a report from it), its tools say how a session stands after they act, and it says what it checked and what it did not; tasks it sends that touch code ask for the checks and tests. It never picks a model: which one is yours to say, as below. What sessions say reaches PiP fenced as data, never as instructions, and what sessions, guests, or GitHub named (titles, waiting calls, scheduled messages, PR titles) comes on one line, quoted, as data too. It makes at most 60 messages, follows, schedules, and watches (its reminders to itself included; a schedule it could not make does not count), and starts at most 20 sessions, an hour (counted since the server started). It can deny a call waiting for approval, never allow one. When one of its own calls waits for approval, the panel says so, and **Review** opens its conversation, where it can be allowed. What it sends shows as sent through PiP for you; it reads files but changes none and runs no commands, handing hands-on work to sessions. Your messages are accent bubbles; completed steps have small green checks, and reports are rounded cards that expand to show more. Its replies link the sessions they name; **Open as a conversation** shows every step, and **Start fresh** gives it a new context. Its context stays small, as PiP needs no long memory of its own: the panel's header measures context against PiP's compaction limit, capped at the model's context window (or just that window with **As sessions do**), and follows changes in Extensions; **Compact** there (or `/compact`, or `/compact` with what to keep in mind, typed to PiP) turns its older messages into a short summary, the newest few kept as they are; and it compacts itself once idle above 60k tokens of context (the owner picks 40k, 60k, 100k, or only as sessions do, in Extensions under PiP; `pipCompactAt` in `~/.config/pi-pocket/config.json`), then waits for 8k tokens more before it does again. A session's thresholds (Pi's own settings) stay as a backstop during a long run. PiP's summary is its own: written from its conversation with what sessions said, webhooks' posts, GitHub's events, and what files and scripts gave back left out, so none of their words come back as PiP's notes; it keeps its person's preferences, what is still open, and what it follows and watches, which live on the server and carry on as they were. A summary that cannot be written leaves the context as it was: Compact's notice says why, and after one of PiP's own that failed it waits 30 minutes, or 8k tokens more, before it tries again. A summary costs a request, counted in PiP's spend. It is a conversation of its own, kept out of the session list and out of everyone else's reach (nobody else can open it or write to it), and durable like the rest: follows outlive a restart. Its cost shows in its person's Spend sheet, and its follows in Running now. Turn it off in Extensions: it stops what it was doing (a compaction too), takes no messages, its own conversation's box included, and meanwhile its follows' reports and its own scheduled messages wait, webhooks to it answer 503, and repository watches to it hold what they found, all going on once it is back. - **Resize.** At widths of 768px and above, drag PiP's left edge, top edge, or top-left corner to resize it, with the bottom-right corner fixed. The same zones are finger-sized on touch screens. Its size is remembered for each account in this browser and fits back inside the viewport when it shrinks. Phones use a bottom sheet with rounded top corners and a grabber; the model, context meter, and all four header actions stay available. - **Choose PiP's model.** Tap the model chip in PiP's header to use the existing model and thinking picker for PiP alone, from Home or over a session. The chip and picker choices are disabled while PiP works. Saving also checks that PiP is idle. This header path does not change the last picked model for new sessions; **Open as a conversation** keeps its existing behavior. With no model, the welcome offers **Choose a model to use these suggestions**, which opens the same picker. Suggestions that send at once wait for a model; those that put text in the box stay usable, and choosing a model keeps the suggestions visible. - **Files and images in chat.** Pick files with the attach button, paste files or images from the clipboard, or drop them into PiP's message box. Remove attachments before sending; upload status shows while they are saved. You can send files with text or on their own. They belong to PiP's conversation, not the session behind the panel. Images have previews and open full screen; other files show their names. As in normal chat, supported images go directly to an image-capable model and other files are given as saved paths. PiP does not choose a different model for you. Draft attachments stay when you close and reopen the panel. @@ -84,10 +84,10 @@ The full tour. For setup, see the [README](../README.md). - **Share to Pi.** With Pi Pocket on the home screen (Android), share a link, some text, or a screenshot from any app to Pi Pocket: pick a session or a new one, and it goes into the message box with the files attached, ready to send. - **Approvals need someone else.** An owner setting under Lancet Guard in Menu → **Extensions**: a guest cannot allow a risky call that their own message led to. Someone else has to; the owner always can. - **Docs for agents.** Pi's system prompt says where Pi Pocket's documentation is ([index.md](index.md)), to read when someone asks about Pi Pocket itself: changing how Pi works without code, writing extensions (with tested examples), and changing Pi Pocket's code while it runs, checked first on a throwaway copy (`node test/serve.ts`). -- **Drop-in extensions.** Put your own extension module (a `.ts` file that default-exports `(host, web) => Extension`, or a list, or nothing) in `~/.pi-pocket/extensions/`. It shows in the Extensions sheet, stays off until the owner turns it on, can import Pi Pocket's packages (Pi Durable, pi-ai, Pi's MCP client pi-mcp, and the rest) as the built-in modules do, and reloads when edited. One that would take the name of a built-in extension or tool, or of another drop-in's, is not loaded, and the sheet says why: installing it would replace that one. +- **Drop-in extensions.** Put your own extension module (a `.ts` file that default-exports `(host, web) => Extension`, or a list, or nothing) in `~/.local/share/pi-pocket/extensions/` (`XDG_DATA_HOME` moves it). It shows in the Extensions sheet, stays off until the owner turns it on, can import Pi Pocket's packages (Pi Durable, pi-ai, Pi's MCP client pi-mcp, and the rest) as the built-in modules do, and reloads when edited. One that would take the name of a built-in extension or tool, or of another drop-in's, is not loaded, and the sheet says why: installing it would replace that one. - **Extensions in the app.** A module can add to the app itself, without changes to Pi Pocket's code: routes on the server, commands for the message box, and a browser module with buttons on the rail, pages in the conversation's place (as the subagents board shows), sheets, rows in the menu, and launcher actions, which work without asking Pi. A dashboard of a provider's usage or the server's load, say: the route asks the service with a key that stays on the server, and the page shows what it says, live with the module's events. It is the owner's alone unless the module opens it to people who can steer, or to everyone who sees every session; someone invited to one session never gets it. Turning the module off takes away what it added, its pages included; an edit swaps the routes, and an edit to the browser module's files has tabs load it again. One that fails tells the owner why in a notice and in the Extensions sheet, and the rest of the app goes on; a tab opened with `?noext=1` loads none of their browser parts (safe mode), to turn off one that breaks the app. [docs/extensions.md](extensions/app.md) says how, with an example: a page with the server's load, memory, and uptime. - **Extension workspace APIs (0.16.0).** A page can save its own durable catalog with request-lifetime document commits, or use a source-bound route to start linked sessions as the person who clicked Run. Current source access, turn, plan, spend, team, and model checks apply; explicit thinking choices are not silently lowered. Human history can open authorized sessions through `pocket.openConversation`. Extensions can read the running server's package version through `host.pocketVersion` or browser `pocket.pocketVersion`, without an updater or restart control; the extension API level remains 1. -- **Extensions hub.** Menu → **Extensions** opens **Discover**, **Installed**, and **Updates** for the owner while the gallery is on. Wide screens show the list and selected entry side by side; phones show one at a time, with Back returning to the filtered list. **Installed** (also tap "guard" in the status line) lists Pi Pocket's extensions with their tools, the routes, commands, and browser module they add to the app, and any load error, in this browser too. The owner can turn Artifacts, Browser, Subagents, Scheduled messages, Done when, Plan mode, Codemode, Memory, PiP, Triggers, Extension gallery, Agent Teams, Lancet Guard, or a drop-in off and on, or reload one; the change applies to every session at once, stays after restarts (`disabledExtensions` and `enabledExtensions` in `~/.pi-pocket/config.json`), and everyone in a session sees who changed what. Turning Lancet Guard off first shows a warning card with **Keep it on** and **Turn off**; only **Turn off** applies the change. This leaves Pi's own setting (`~/.pi/lancet-guard.json`) alone. Under PiP, the owner picks when PiP compacts its context. The system prompt cannot be turned off. +- **Extensions hub.** Menu → **Extensions** opens **Discover**, **Installed**, and **Updates** for the owner while the gallery is on. Wide screens show the list and selected entry side by side; phones show one at a time, with Back returning to the filtered list. **Installed** (also tap "guard" in the status line) lists Pi Pocket's extensions with their tools, the routes, commands, and browser module they add to the app, and any load error, in this browser too. The owner can turn Artifacts, Browser, Subagents, Scheduled messages, Done when, Plan mode, Codemode, Memory, PiP, Triggers, Extension gallery, Agent Teams, Lancet Guard, or a drop-in off and on, or reload one; the change applies to every session at once, stays after restarts (`disabledExtensions` and `enabledExtensions` in `~/.config/pi-pocket/config.json`), and everyone in a session sees who changed what. Turning Lancet Guard off first shows a warning card with **Keep it on** and **Turn off**; only **Turn off** applies the change. This leaves Pi's own setting (`~/.pi/lancet-guard.json`) alone. Under PiP, the owner picks when PiP compacts its context. The system prompt cannot be turned off. - **Extension gallery.** Menu → Extensions → **Discover** (or **Find extensions** in the launcher) lists drop-ins other people wrote, from a reviewed index ([TannerMidd/pi-pocket-extensions](https://github.com/TannerMidd/pi-pocket-extensions) unless the owner picks another): search them by name, words, and tags; entries for a newer Pi Pocket are hidden. **Install** is available on each list card. Details put the main actions first, with technical information folded below. An entry shows who wrote it, where its code is (a GitHub repository at a pinned commit), its files, and what it declares: its tools (and which only read), hooks, prompt sections, wraps, routes, and browser module, and who can reach them. **Install** downloads its files over https from that commit, checks each against the SHA-256 the index pins (one that differs installs nothing), and puts them in the extensions folder, off, as every drop-in starts. Before turning it on, read its whole source in the gallery, or **Ask Pi to review**: a new session in the extensions folder, in plan mode, where Pi audits what the code reads (`config.json`, sign-ins, other secrets), the addresses it calls, the processes it starts, what its browser module does, and whether it does what it declares. Updates are never automatic: the **Updates** tab lists available updates with a **Review update** button, and the review shows what changes, as a diff of every file and of what it declares, before you apply it. When the index revokes the version you have, the gallery and Extensions say why and offer to turn it off, and a notice says so while it is on. **Uninstall** removes its files (where each came from is kept in `.gallery.json` beside it). **Share…** on one of your own drop-ins in Extensions checks that it stands on its own (its files named after it, importing only Pi Pocket's packages and its own files), drafts its index entry (the description from its doc comment, each file's SHA-256, what it declares while it is on), asks where its files are on GitHub (the repository and commit you put them in, tags, your GitHub name), and, once the entry would pass the index's checks, opens GitHub's page for a pull request to the index with the entry filled in; Pi Pocket itself makes no GitHub calls. [docs/extensions.md](extensions/publishing.md) says how to publish. The owner's alone; turned off in Extensions, it reads nothing. It lists themes too, with a **Theme** tag and filter: **Install** checks a theme's pinned files and its CSS before it goes into `themes/`, and puts it in Appearance's list without applying it; **Update** shows a diff; **Remove** moves it to `themes/.removed/`; **Share…** in Appearance drafts an entry for one of your own themes ([themes.md](extensions/themes.md#themes-in-the-gallery)). - **Refresh models.** The owner can tap **Refresh models** in the menu (under App) or the Providers sheet (above Models for subagents), or run **Refresh models** from the Ctrl/⌘+K launcher. The same controls work on phones. Refresh reloads `models.json` and Pi’s sign-ins, and asks configured providers for fresh catalogs. The menu's row says when it is refreshing, and a notice gives the added and removed model counts, without closing the menu; the Providers sheet also shows the errors for individual providers. All connected tabs receive the updated list. Healthy providers finish independently of slow ones; each provider has an eight-second limit. Running sessions keep their chosen models. A local model must be configured in `models.json` or registered by a Pocket-compatible extension: refresh does not load regular Pi CLI extensions or discover arbitrary local servers. - **Uploads.** In normal chat and PiP, attach files with the picker, paste files or images, or drop them into the message box. Files are saved on the server for the conversation you send to and their paths are given to the agent; supported images also go straight to models that accept them. diff --git a/docs/index.md b/docs/index.md index 90c36285..513e865d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,11 +5,11 @@ These pages are for agents and people who change how Pi works in Pi Pocket, exte ## Where things are - **The code** is in the folder above this one (`web/`, `src/`, `docs/`, `test/`). It runs directly from these files: there is no build. -- **The data** is in `~/.pi-pocket/`, or in `PI_POCKET_DIR`. The launcher shows it as **Data**. - - `config.json` has the people, roles, tokens, invites, and settings. +- **The data** is in `~/.local/share/pi-pocket/` (`XDG_DATA_HOME` moves it), or in `PI_POCKET_DATA_DIR`. The launcher shows it as **Data**. - `pocket.sqlite` has all sessions. - `extensions/` has the owner's drop-in extensions. - `themes/` has the owner's themes. + - `config.json`, in `~/.config/pi-pocket/` (`PI_POCKET_CONFIG_DIR`), has the people, roles, tokens, invites, and settings. The state and cache folders (`~/.local/state/pi-pocket/`, `~/.cache/pi-pocket/`) hold the lock and tunnel log, and the downloaded cloudflared and browser profile. - Change the data through the app, never in these files: the server keeps them in memory, and writes over them. There are two exceptions: `extensions/`, where drop-ins go ([extensions.md](extensions.md)), and `themes/`, where themes go ([extensions/themes.md](extensions/themes.md)). - **Pi's own files** are in `~/.pi/agent/`: sign-ins, models, settings, `AGENTS.md`, skills, and prompt templates. Pi Pocket uses them as Pi does. It only reads Pi's sessions (`sessions/`), so that the owner can continue one here. @@ -17,15 +17,15 @@ These pages are for agents and people who change how Pi works in Pi Pocket, exte Each step goes further than the step before it, and needs more care. Use the first one that does the job. -| To change | Use | Read | -| ------------------------------------------------------------------ | ------------------------------------------------------------------------------- | -------------------------------------------- | -| How Pi works in one session | The session's instructions (Menu → Instructions for Pi) | [customizing.md](customizing.md) | -| How Pi works in a project, or everywhere | An `AGENTS.md` in the project, or Pi's own in `~/.pi/agent/` | [customizing.md](customizing.md) | -| A procedure that Pi follows on request | A skill, or a prompt template: both become slash commands | [customizing.md](customizing.md) | -| What Pi can do: a new tool, a rule in its prompt, a check on calls | A drop-in extension in `~/.pi-pocket/extensions/`, which the owner turns on | [extensions.md](extensions.md) | -| What the app shows: a page, a rail button, a route of your own | The same drop-in, with a web part | [extensions/app.md](extensions/app.md) | -| How the app looks and is laid out | A theme: a folder of CSS in `~/.pi-pocket/themes/`, from Appearance → Duplicate | [extensions/themes.md](extensions/themes.md) | -| Pi Pocket itself: its screens, its server, a built-in extension | Its code, while it runs | [self-editing.md](self-editing.md) | +| To change | Use | Read | +| ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------- | -------------------------------------------- | +| How Pi works in one session | The session's instructions (Menu → Instructions for Pi) | [customizing.md](customizing.md) | +| How Pi works in a project, or everywhere | An `AGENTS.md` in the project, or Pi's own in `~/.pi/agent/` | [customizing.md](customizing.md) | +| A procedure that Pi follows on request | A skill, or a prompt template: both become slash commands | [customizing.md](customizing.md) | +| What Pi can do: a new tool, a rule in its prompt, a check on calls | A drop-in extension in `~/.local/share/pi-pocket/extensions/`, which the owner turns on | [extensions.md](extensions.md) | +| What the app shows: a page, a rail button, a route of your own | The same drop-in, with a web part | [extensions/app.md](extensions/app.md) | +| How the app looks and is laid out | A theme: a folder of CSS in `~/.local/share/pi-pocket/themes/`, from Appearance → Duplicate | [extensions/themes.md](extensions/themes.md) | +| Pi Pocket itself: its screens, its server, a built-in extension | Its code, while it runs | [self-editing.md](self-editing.md) | A drop-in extension stays when Pi Pocket updates, and the owner can turn it off in the app. A change to Pi Pocket's code is for each person who uses this copy, and can break it for all of them. diff --git a/docs/map.md b/docs/map.md index 6daf2041..c9a8b69f 100644 --- a/docs/map.md +++ b/docs/map.md @@ -6,11 +6,14 @@ Where things are in Pi Pocket, for agents working on it. [index.md](index.md) li `bin/pi-pocket.js` checks Node, then runs `src/launcher/main.ts`. The launcher reads its options (`options.ts`), asks how devices connect (`access.ts`: this device, LAN, Cloudflare quick tunnel, Tailscale), runs the server (`src/server/main.ts`) as a child, restarts it on exit code 75 or a crash, and keeps the tunnel up. `term.ts` draws its menu, panel, and QR code in the terminal. The server takes the data folder's lock, opens one `PocketApp`, and serves it with `http.ts`. -## Data (`~/.pi-pocket/`, or `PI_POCKET_DIR`) +## Data (the XDG folders, or `PI_POCKET_*_DIR`) -- `pocket.sqlite`: Pi Durable's storage: conversations, entries, tasks, and Pi Pocket's documents. -- `config.json`: people, roles, hashed tokens, settings. `push.json`: VAPID keys and push subscriptions. -- `uploads//`, `worktrees/` (sessions' git worktrees), `extensions/` (the owner's drop-ins), `themes/` (the owner's theme folders, and `.removed/` for removed ones), `browser/profile/` (the built-in browser's profile; each session's cookies live in memory only). +`dirs.ts` works out the four folders (config, data, state, cache) from `XDG_*_HOME` and moves an older `~/.pi-pocket` into them once; the launcher passes them to the server as `PI_POCKET_*_DIR`. + +- `config.json`: people, roles, hashed tokens, settings. `launcher.json`: the launcher's last access choice. Both in the config folder (`~/.config/pi-pocket/`). +- `pocket.sqlite`: Pi Durable's storage: conversations, entries, tasks, and Pi Pocket's documents. `push.json`: VAPID keys and push subscriptions. `harness.lock`: the process id of the server using the folder, so one runs there at a time. +- `uploads//`, `worktrees/` (sessions' git worktrees), `extensions/` (the owner's drop-ins), `themes/` (the owner's theme folders, and `.removed/` for removed ones): all in the data folder. +- `cloudflared.log` in the state folder (`~/.local/state/pi-pocket/`); `bin/cloudflared` and `browser/profile/` (the built-in browser's profile; each session's cookies live in memory only) in the cache folder (`~/.cache/pi-pocket/`). ## Server: `src/server/` diff --git a/nix/module.nix b/nix/module.nix index cb6b430b..70d0cfef 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -52,7 +52,11 @@ in dataDir = lib.mkOption { type = lib.types.path; default = "/var/lib/pi-pocket"; - description = "Directory for the database, settings, uploads, and extensions."; + description = '' + Directory for the database, settings, uploads, and extensions. The service keeps + Pi Pocket's four folders here as well, so an existing deployment's files stay where + they are instead of moving into the user's XDG folders. + ''; }; cwd = lib.mkOption { @@ -234,6 +238,10 @@ in ''; environment = { HOME = config.users.users.${cfg.user}.home; + PI_POCKET_CONFIG_DIR = cfg.dataDir; + PI_POCKET_DATA_DIR = cfg.dataDir; + PI_POCKET_STATE_DIR = cfg.dataDir; + PI_POCKET_CACHE_DIR = cfg.dataDir; } // lib.optionalAttrs (cfg.browser != null) { PI_POCKET_BROWSER = cfg.browser; } // lib.optionalAttrs (cfg.browserArgs != [ ]) { @@ -252,8 +260,6 @@ in cfg.access "--port" (toString cfg.port) - "--data" - cfg.dataDir "--cwd" cfg.cwd ] diff --git a/nix/tests/extension.ts b/nix/tests/extension.ts index b68e75f3..ece3cf23 100644 --- a/nix/tests/extension.ts +++ b/nix/tests/extension.ts @@ -4,7 +4,7 @@ import { Type } from "@earendil-works/pi-ai"; import { defineExtension, defineTool } from "@earendil-works/pi-durable"; export default () => { - const marker = join(process.env.PI_POCKET_DIR!, "runtime-probe.json"); + const marker = join(process.env.PI_POCKET_DATA_DIR!, "runtime-probe.json"); // both runners poll this marker while the extension loads writeFileSync(`${marker}.tmp`, JSON.stringify({ pid: process.pid, version: "v1" })); diff --git a/nix/tests/installed.mjs b/nix/tests/installed.mjs index d06411f7..a385d5f5 100644 --- a/nix/tests/installed.mjs +++ b/nix/tests/installed.mjs @@ -26,9 +26,10 @@ test( const source = readFileSync(process.env.PI_POCKET_TEST_EXTENSION, "utf8"); const root = mkdtempSync(join(tmpdir(), "pi-pocket-installed-")); const data = join(root, "data"); + const config = join(root, "config"); const extensions = join(data, "extensions"); const file = join(extensions, "extension.ts"); - const jsonFile = (name) => JSON.parse(readFileSync(join(data, name), "utf8")); + const jsonFile = (folder, name) => JSON.parse(readFileSync(join(folder, name), "utf8")); mkdirSync(extensions, { recursive: true }); writeFileSync(file, source); @@ -50,18 +51,7 @@ test( function start() { child = spawn( join(packageDir, "bin/pi-pocket"), - [ - "--access", - "local", - "--host", - "127.0.0.1", - "--port", - String(port), - "--data", - data, - "--cwd", - root, - ], + ["--access", "local", "--host", "127.0.0.1", "--port", String(port), "--cwd", root], { cwd: root, detached: true, @@ -73,6 +63,10 @@ test( TMPDIR: tmpdir(), PI_CODING_AGENT_DIR: join(root, "agent"), PI_POCKET_GUARD: "off", + PI_POCKET_CONFIG_DIR: config, + PI_POCKET_DATA_DIR: data, + PI_POCKET_STATE_DIR: join(root, "state"), + PI_POCKET_CACHE_DIR: join(root, "cache"), }, }, ); @@ -148,7 +142,7 @@ test( const response = await fetch(`${base}/api/${path}`, { method: body === undefined ? "GET" : "POST", headers: { - Authorization: `Bearer ${jsonFile("config.json").ownerToken}`, + Authorization: `Bearer ${jsonFile(config, "config.json").ownerToken}`, "X-Pocket": "1", "Content-Type": "application/json", }, @@ -183,12 +177,12 @@ test( assert.deepEqual(current.extensions, [ { name: `runtime-${version}`, tools: [`probe_${version}`] }, ]); - assert.equal(jsonFile("runtime-probe.json").version, version); + assert.equal(jsonFile(data, "runtime-probe.json").version, version); } start(); await ready(); - const token = jsonFile("config.json").ownerToken; + const token = jsonFile(config, "config.json").ownerToken; const session = await api("sessions", { title: "Installed runtime session", }); @@ -232,12 +226,12 @@ test( await api("extensions/extension.ts", { enabled: true }); await loaded("v1"); - const serverPid = jsonFile("runtime-probe.json").pid; + const serverPid = jsonFile(data, "runtime-probe.json").pid; writeFileSync(file, source.replaceAll("v1", "v2")); await loaded("v2"); assert.equal( - jsonFile("runtime-probe.json").pid, + jsonFile(data, "runtime-probe.json").pid, serverPid, "hot reload does not restart the server", ); @@ -245,7 +239,7 @@ test( // Restart through Pocket's actual supervisor, not by reopening an in-process app. await api("restart", {}); await until( - () => jsonFile("runtime-probe.json").pid !== serverPid, + () => jsonFile(data, "runtime-probe.json").pid !== serverPid, "the supervisor to restart the server", ); await ready(); @@ -257,7 +251,7 @@ test( await ready(); await loaded("v2"); assert.equal(readFileSync(file, "utf8"), source.replaceAll("v1", "v2")); - assert.equal(jsonFile("config.json").ownerToken, token); + assert.equal(jsonFile(config, "config.json").ownerToken, token); assert.ok( (await api("sessions")).some( (each) => each.id === session.id && each.title === "Installed runtime session", diff --git a/site/architecture.html b/site/architecture.html index 1d541243..195ac77a 100644 --- a/site/architecture.html +++ b/site/architecture.html @@ -1888,7 +1888,7 @@

Data storage

- Outside the database, in ~/.pi-pocket/ + Outside the database, in Pi Pocket's folders

@@ -2441,7 +2441,7 @@

Live reload and restart

}, sqlite: { name: "SQLite", - path: "~/.pi-pocket/pocket.sqlite", + path: "~/.local/share/pi-pocket/pocket.sqlite", c: "var(--commit-ink)", main: true, text: "The database is one file in WAL mode. Only one server process can use it at a time. Thus, the server first gets the lock on the data folder.", diff --git a/site/engineering.html b/site/engineering.html index a7d2192e..9f08e17f 100644 --- a/site/engineering.html +++ b/site/engineering.html @@ -158,7 +158,7 @@

The parts.

one file
- SQLite~/.pi-pocket/pocket.sqlite + SQLite~/.local/share/pi-pocket/pocket.sqlite
@@ -598,9 +598,11 @@

Outside the database

harness.lock one server per data folder

- All of it is in ~/.pi-pocket/, or wherever - PI_POCKET_DIR points. The config files are written atomically - with mode 0600. + All of it is in Pi Pocket's XDG folders: + ~/.local/share/pi-pocket/, ~/.config/pi-pocket/, + ~/.local/state/pi-pocket/, and + ~/.cache/pi-pocket/, or wherever PI_POCKET_*_DIR + points. The config files are written atomically with mode 0600.

@@ -702,8 +704,9 @@

Features are extensions.

- The owner's own modules in ~/.pi-pocket/extensions/ load after - these, stay off until turned on, and import the same packages through a + The owner's own modules in + ~/.local/share/pi-pocket/extensions/ load after these, stay off + until turned on, and import the same packages through a node_modules link.

diff --git a/site/index.html b/site/index.html index ff0bf90a..1cdcd576 100644 --- a/site/index.html +++ b/site/index.html @@ -147,7 +147,7 @@

Built on Pi Durable.

saves each step, resumes runs after a restart, queues messages, and runs subagents. Pi Pocket adds the screens, the people, and the ways to connect. Its sessions live in one SQLite file, - ~/.pi-pocket/pocket.sqlite. + ~/.local/share/pi-pocket/pocket.sqlite.

Needs a restart

- Your own extensions go in ~/.pi-pocket/extensions/. They load - after the built-in ones, reload when edited, and stay off until the owner - turns them on in Menu → Extensions. + Your own extensions go in ~/.local/share/pi-pocket/extensions/. + They load after the built-in ones, reload when edited, and stay off until + the owner turns them on in Menu → Extensions.

diff --git a/src/launcher/access.ts b/src/launcher/access.ts index 781d7d2f..a68069ae 100644 --- a/src/launcher/access.ts +++ b/src/launcher/access.ts @@ -151,9 +151,9 @@ function findExecutable(name: string, extraDirectories: string[]): string | unde return undefined; } -/** cloudflared from PATH, or the copy the launcher downloaded into the data folder. */ -export function findCloudflared(dataDir: string): string | undefined { - return findExecutable("cloudflared", [join(dataDir, "bin")]); +/** cloudflared from PATH, or the copy the launcher downloaded into the cache folder. */ +export function findCloudflared(cacheDir: string): string | undefined { + return findExecutable("cloudflared", [join(cacheDir, "bin")]); } /** Where to get cloudflared on this platform: a release binary the launcher can download, or how to install it. */ @@ -186,13 +186,13 @@ export function cloudflaredSource(): { url: string } | { install: string } { }; } -/** Download Cloudflare's release binary into `/bin/cloudflared` and check that it runs. */ +/** Download Cloudflare's release binary into `/bin/cloudflared` and check that it runs. */ export async function downloadCloudflared( - dataDir: string, + cacheDir: string, url: string, progress: (done: number, total: number) => void, ): Promise { - const directory = join(dataDir, "bin"); + const directory = join(cacheDir, "bin"); mkdirSync(directory, { recursive: true, mode: 0o700 }); const target = join(directory, "cloudflared"); diff --git a/src/launcher/main.ts b/src/launcher/main.ts index f0aa324f..4a842b9a 100644 --- a/src/launcher/main.ts +++ b/src/launcher/main.ts @@ -3,15 +3,16 @@ * quick tunnel, or Tailscale), then it runs the server and the tunnel. The server restarts when the app asks (exit code * 75) or after a crash; the tunnel stays up across those restarts, so its address does not change. * - * pi-pocket [--access local|lan|cloudflare|tailscale] [-y] [--port 8787] [--cwd DIR] [--data DIR] [--rotate-token] + * pi-pocket [--access local|lan|cloudflare|tailscale] [-y] [--port 8787] [--cwd DIR] [--rotate-token] */ import { type ChildProcess, spawn } from "node:child_process"; import { once } from "node:events"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { createInterface } from "node:readline"; import { fileURLToPath } from "node:url"; +import { ensureFolders, foldersFromEnv, migrateLegacy } from "../server/dirs.ts"; import { ACCESS_LABELS, type AccessMode, @@ -52,8 +53,13 @@ const RESTART_CODE = 75; const options = parseArgs(process.argv.slice(2)); -mkdirSync(options.data, { recursive: true, mode: 0o700 }); -const settingsFile = join(options.data, "launcher.json"); +// The four folders, in the XDG base directories unless PI_POCKET_*_DIR names them: a `~/.pi-pocket` from an older +// Pi Pocket moves into them once, before anything reads it. +const dirs = foldersFromEnv(); + +ensureFolders(dirs); + +const settingsFile = join(dirs.config, "launcher.json"); function loadSettings(): { access?: AccessMode } { try { @@ -112,7 +118,7 @@ function accessChoices(): { mode: AccessMode; choice: Choice }[] { const port = options.port; const lan = lanAddresses()[0]; const tailnet = tailscaleAddress(); - const cloudflared = findCloudflared(options.data); + const cloudflared = findCloudflared(dirs.cache); const source = cloudflaredSource(); return [ @@ -195,7 +201,7 @@ async function pickAccess(current: AccessMode | undefined): Promise { - const found = findCloudflared(options.data); + const found = findCloudflared(dirs.cache); if (found !== undefined) { return found; @@ -221,7 +227,7 @@ async function ensureCloudflared(): Promise { log(""); const yes = await confirm( - `Download cloudflared (Cloudflare's official release, about 40 MB) to ${tilde(join(options.data, "bin"))}?`, + `Download cloudflared (Cloudflare's official release, about 40 MB) to ${tilde(join(dirs.cache, "bin"))}?`, true, ); @@ -232,7 +238,7 @@ async function ensureCloudflared(): Promise { let last = 0; try { - const file = await downloadCloudflared(options.data, source.url, (done, total) => { + const file = await downloadCloudflared(dirs.cache, source.url, (done, total) => { if (Date.now() - last < 120 && done !== total) { return; } @@ -381,7 +387,7 @@ async function showPanel(withQr: boolean): Promise { } lines.push(row("Folder", tilde(options.cwd))); - lines.push(row("Data", tilde(options.data))); + lines.push(row("Data", tilde(dirs.data))); lines.push(""); const login = state.loginPath === undefined @@ -446,8 +452,6 @@ function startServer(first: boolean): void { String(options.port), "--cwd", options.cwd, - "--data", - options.data, ]; if (first && options.rotateToken) { @@ -455,9 +459,18 @@ function startServer(first: boolean): void { } // Node 22 calls node:sqlite experimental and says so at every start, right above the panel. + // The four folders come along as the variables the server reads, so launcher and server always agree. const proc = spawn(process.execPath, ["--disable-warning=ExperimentalWarning", ...args], { stdio: ["ignore", "pipe", "pipe", "ipc"], - env: { ...process.env, PI_POCKET_SUPERVISED: "1", PI_POCKET_LAUNCHER: "1" }, + env: { + ...process.env, + PI_POCKET_SUPERVISED: "1", + PI_POCKET_LAUNCHER: "1", + PI_POCKET_CONFIG_DIR: dirs.config, + PI_POCKET_DATA_DIR: dirs.data, + PI_POCKET_STATE_DIR: dirs.state, + PI_POCKET_CACHE_DIR: dirs.cache, + }, }); child = proc; @@ -579,7 +592,7 @@ function startTunnel(binary: string): void { const current: CloudflareTunnel = new CloudflareTunnel( binary, options.port, - join(options.data, "cloudflared.log"), + join(dirs.state, "cloudflared.log"), { ready(url) { if (tunnel !== current) { @@ -619,7 +632,7 @@ function startTunnel(binary: string): void { log( yellow( - ` The tunnel stopped (code ${code}); trying again in ${delay / 1000} seconds. Log: ${tilde(join(options.data, "cloudflared.log"))}`, + ` The tunnel stopped (code ${code}); trying again in ${delay / 1000} seconds. Log: ${tilde(join(dirs.state, "cloudflared.log"))}`, ), ); maybeShowPanel(); @@ -778,6 +791,7 @@ process.on("exit", () => restoreTerminal()); async function main(): Promise { // Until the server runs, Ctrl+C (raw mode turns off the terminal's own) still quits, even mid-download. onKeys((key) => key === "ctrl-c" && void quit(130)); + migrateLegacy(dirs, log); const settings = loadSettings(); let mode: Mode | undefined = options.access; diff --git a/src/launcher/options.ts b/src/launcher/options.ts index f3109696..8c1a66aa 100644 --- a/src/launcher/options.ts +++ b/src/launcher/options.ts @@ -1,6 +1,5 @@ /** The launcher's command line: its options, from arguments and the environment, and its help. */ -import { homedir } from "node:os"; -import { join, resolve } from "node:path"; +import { resolve } from "node:path"; import { ACCESS_MODES, type AccessMode, isAccessMode } from "./access.ts"; import { restoreTerminal } from "./term.ts"; @@ -17,12 +16,15 @@ Usage: pi-pocket [options] -y, --yes Use the last choice without asking -p, --port N Port (default 8787) --cwd DIR Folder for new sessions (default: the current folder) - --data DIR Data folder (default ~/.pi-pocket) --host HOST Listen on this address instead of choosing an access mode --rotate-token Replace the owner sign-in link, signing out the devices that used it -h, --help Show this help -Environment: PI_POCKET_ACCESS, PI_POCKET_PORT, PI_POCKET_DIR, PI_POCKET_HOST, PI_POCKET_GUARD=off`; +Environment: PI_POCKET_ACCESS, PI_POCKET_PORT, PI_POCKET_HOST, PI_POCKET_GUARD=off, and the folders: + PI_POCKET_CONFIG_DIR (default XDG_CONFIG_HOME/pi-pocket, ~/.config/pi-pocket) + PI_POCKET_DATA_DIR (default XDG_DATA_HOME/pi-pocket, ~/.local/share/pi-pocket) + PI_POCKET_STATE_DIR (default XDG_STATE_HOME/pi-pocket, ~/.local/state/pi-pocket) + PI_POCKET_CACHE_DIR (default XDG_CACHE_HOME/pi-pocket, ~/.cache/pi-pocket)`; export interface Options { access: AccessMode | undefined; @@ -30,7 +32,6 @@ export interface Options { host: string | undefined; port: number; cwd: string; - data: string; rotateToken: boolean; } @@ -42,7 +43,6 @@ export function parseArgs(argv: string[]): Options { host: process.env.PI_POCKET_HOST || undefined, port: Number(process.env.PI_POCKET_PORT ?? 8787), cwd: process.cwd(), - data: resolve(process.env.PI_POCKET_DIR ?? join(homedir(), ".pi-pocket")), rotateToken: false, }; @@ -75,8 +75,6 @@ export function parseArgs(argv: string[]): Options { options.port = Number(value()); } else if (arg === "--cwd") { options.cwd = resolve(value()); - } else if (arg === "--data") { - options.data = resolve(value()); } else if (arg === "--rotate-token") { options.rotateToken = true; } else if (arg === "--help" || arg === "-h") { diff --git a/src/server/app.ts b/src/server/app.ts index 1dfde256..88d54d5b 100644 --- a/src/server/app.ts +++ b/src/server/app.ts @@ -48,6 +48,7 @@ import { Collab, REACTIONS } from "./collab.ts"; import { Commands } from "./commands.ts"; import { LinkedSessions } from "./linked-sessions.ts"; import { APP_ROOT, ConfigStore, type User } from "./config.ts"; +import { ensureFolders, type Folders } from "./dirs.ts"; import { ArtifactBodyDoc, type ArtifactMeta, @@ -234,7 +235,7 @@ function countSubagents( } export interface OpenOptions { - dataDir: string; + dirs: Folders; defaultCwd: string; supervised: boolean; log?: (line: string) => void; @@ -254,7 +255,17 @@ export class PocketApp { pushStore: PushStore | undefined; readonly approvals = new Approvals((id) => this.attribution.requesterOf(id)); readonly guard = new LancetGuard(); + /** + * The data folder: the database and its lock, uploads, worktrees, gallery, themes, and drop-ins. Also + * `host.dataDir`. + */ readonly dataDir: string; + /** `config.json` and `launcher.json`: people, roles, tokens, settings, and the launcher's last choice. */ + readonly configDir: string; + /** The tunnel's log: the one thing Pi Pocket writes that is only worth keeping while it runs. */ + readonly stateDir: string; + /** What Pi Pocket can download again: cloudflared and the built-in browser's profile. */ + readonly cacheDir: string; readonly defaultCwd: string; readonly supervised: boolean; readonly startedAt = Date.now(); @@ -345,18 +356,27 @@ export class PocketApp { /** The clock durable work runs by. */ readonly now: () => number; + /** What Pi's own skills and settings are read from, and where an older Pi Pocket's folder may still be. */ + get homeFolder(): string { + return this.#home; + } + private constructor(options: OpenOptions) { this.#configureModels = options.configureModels; this.#home = options.home ?? (process.env.HOME || homedir()); this.now = options.now ?? Date.now; - this.dataDir = options.dataDir; + this.dataDir = options.dirs.data; + this.configDir = options.dirs.config; + this.stateDir = options.dirs.state; + this.cacheDir = options.dirs.cache; this.defaultCwd = options.defaultCwd; this.supervised = options.supervised; - this.config = new ConfigStore(options.dataDir); - this.#lockFile = join(options.dataDir, "harness.lock"); + this.config = new ConfigStore(options.dirs.config); + // In the data folder, not the state folder: the lock holds a database, and says which folder is taken. + this.#lockFile = join(options.dirs.data, "harness.lock"); this.#log = options.log ?? ((line) => console.log(line)); this.browsers = new Browsers({ - dataDir: options.dataDir, + cacheDir: options.dirs.cache, ...(options.browser === undefined ? {} : { executable: options.browser }), log: (line) => this.#log(line), load: async (id) => @@ -390,7 +410,7 @@ export class PocketApp { }); this.themes = new Themes({ builtIn: join(APP_ROOT, "web", "themes"), - dataDir: options.dataDir, + dataDir: options.dirs.data, notify: (message) => this.ownerNotice("warning", message), // Everyone's tabs: a theme anyone may have chosen changed, or one came or went. changed: () => { @@ -472,7 +492,17 @@ export class PocketApp { return app; } + /** + * The four folders first: the config folder holds `config.json`, the data folder the database and the lock taken + * below, and the launcher and the server make them; a caller with its own (as tests do) gets them too. + */ async #open(): Promise { + ensureFolders({ + config: this.configDir, + data: this.dataDir, + state: this.stateDir, + cache: this.cacheDir, + }); takeLock(this.#lockFile); try { diff --git a/src/server/browser.ts b/src/server/browser.ts index 7bfaee71..659387fc 100644 --- a/src/server/browser.ts +++ b/src/server/browser.ts @@ -25,8 +25,8 @@ export type OpenOptions = { }; export interface BrowsersOptions { - /** Where the browser's profile goes: `browser/` in it. */ - dataDir: string; + /** Where the browser's profile goes: `browser/profile/` in it. */ + cacheDir: string; /** The browser to run. Undefined finds one; null means there is none (tests). */ executable?: string | null; /** Extra Chromium flags, such as `--no-sandbox` where sandboxes are not available. */ @@ -164,7 +164,7 @@ export class Browsers { ]; const launching = Chromium.launch( executable, - profileFolder(executable, this.#options.dataDir), + profileFolder(executable, this.#options.cacheDir), extra, ); diff --git a/src/server/browser/discovery.ts b/src/server/browser/discovery.ts index 2fb2bbcc..3fd353a1 100644 --- a/src/server/browser/discovery.ts +++ b/src/server/browser/discovery.ts @@ -95,17 +95,17 @@ export function snapOf( } /** - * Where the browser keeps its profile. A snap may not write to hidden folders in the home folder, as the data folder - * usually is (`~/.pi-pocket`): its profile goes in the snap's own folder instead, one per data folder. + * Where the browser keeps its profile. A snap may not write to hidden folders in the home folder, as the cache folder + * usually is (`~/.cache`): its profile goes in the snap's own folder instead, one per cache folder. */ -export function profileFolder(executable: string, dataDir: string, snapBin = SNAP_BIN): string { +export function profileFolder(executable: string, cacheDir: string, snapBin = SNAP_BIN): string { const snap = snapOf(executable, snapBin); if (snap === undefined) { - return join(dataDir, "browser", "profile"); + return join(cacheDir, "browser", "profile"); } - const id = createHash("sha256").update(dataDir).digest("hex").slice(0, 12); + const id = createHash("sha256").update(cacheDir).digest("hex").slice(0, 12); return join(homedir(), "snap", snap.name, "common", "pi-pocket", id); } diff --git a/src/server/config.ts b/src/server/config.ts index 90a5a2d4..8c2a97c2 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -1,17 +1,11 @@ import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; -import { homedir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; /** The repository root: `web/`, `src/`, and `node_modules/` live here. */ export const APP_ROOT = fileURLToPath(new URL("../..", import.meta.url)); -/** Where Pi Pocket keeps its database, users, uploads, and logs. */ -export function dataDir(): string { - return process.env.PI_POCKET_DIR ?? join(homedir(), ".pi-pocket"); -} - /** owner: everything. guest: steers Pi (which can run commands here). viewer: reads, chats, and reacts, never steers. */ export type Role = "owner" | "guest" | "viewer"; @@ -105,7 +99,7 @@ function sameHash(a: string, b: string): boolean { /** The most invites one person keeps live at once. */ export const MAX_INVITES = 50; -/** `config.json` in the data directory, written atomically with mode 0600. */ +/** `config.json` in the config folder, written atomically with mode 0600. */ export class ConfigStore { readonly file: string; #config: PocketConfig; diff --git a/src/server/dirs.ts b/src/server/dirs.ts new file mode 100644 index 00000000..9c5c7344 --- /dev/null +++ b/src/server/dirs.ts @@ -0,0 +1,231 @@ +/** + * Pi Pocket's folders: which parts of its state live where, per the XDG base directory spec. The launcher and the + * server both work them out here, so they always agree, and an older `~/.pi-pocket` moves into them once. + */ +import { + chmodSync, + cpSync, + existsSync, + mkdirSync, + readFileSync, + readdirSync, + renameSync, + rmSync, + rmdirSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { isAbsolute, join, resolve, sep } from "node:path"; +import { lockHolder } from "./lock.ts"; + +/** Pi Pocket's folders: which parts of its state live where, per the XDG base directory spec. */ +export interface Folders { + config: string; + data: string; + state: string; + cache: string; +} + +/** What each kind of folder holds, as an old `~/.pi-pocket` named it. */ +const LEGACY_ENTRIES: Record = { + config: ["config.json", "launcher.json"], + data: [ + "pocket.sqlite", + "pocket.sqlite-wal", + "pocket.sqlite-shm", + "push.json", + // Moved with the database it guards: a server still on the old layout is then refused, not doubled. + "harness.lock", + "uploads", + "worktrees", + "repos", + "gallery", + "themes", + "extensions", + ], + state: ["cloudflared.log"], + cache: ["bin", "browser"], +}; + +/** What the app stores more than once: uploads a message carries, and the folders sessions work in. */ +const NAMED_BY_PATH = new Set(["uploads", "worktrees", "repos"]); + +/** An XDG variable: an absolute value, else the default under the home folder (the spec ignores a relative one). */ +function xdg(env: NodeJS.ProcessEnv, name: string, home: string, fallback: string): string { + const value = env[name]; + + return value !== undefined && value !== "" && isAbsolute(value) ? value : join(home, fallback); +} + +/** + * The four folders: `PI_POCKET_*_DIR`, else the XDG folder, else the XDG default under the home folder. An override + * is a person's own choice, so a relative one counts, from the working folder. + */ +export function foldersFromEnv( + env: NodeJS.ProcessEnv = process.env, + home: string = homedir(), +): Folders { + const pick = (name: string, base: string): string => { + const value = env[name]; + + return value !== undefined && value !== "" ? resolve(value) : join(base, "pi-pocket"); + }; + + return { + config: pick("PI_POCKET_CONFIG_DIR", xdg(env, "XDG_CONFIG_HOME", home, ".config")), + data: pick("PI_POCKET_DATA_DIR", xdg(env, "XDG_DATA_HOME", home, join(".local", "share"))), + state: pick( + "PI_POCKET_STATE_DIR", + xdg(env, "XDG_STATE_HOME", home, join(".local", "state")), + ), + cache: pick("PI_POCKET_CACHE_DIR", xdg(env, "XDG_CACHE_HOME", home, ".cache")), + }; +} + +/** Create the four, 0700, as private as the old data folder was: tokens and keys live in them. */ +export function ensureFolders(dirs: Folders): void { + for (const folder of Object.values(dirs)) { + mkdirSync(folder, { recursive: true, mode: 0o700 }); + // `mode` reaches only folders this made: one named with `PI_POCKET_*_DIR` may have come another way. + chmodSync(folder, 0o700); + } +} + +/** `~` for the home folder, as the launcher's and the server's messages show paths. */ +function tilde(folder: string, home: string): string { + return folder === home || folder.startsWith(`${home}${sep}`) + ? `~${folder.slice(home.length)}` + : folder; +} + +/** + * Move one entry, across folders or filesystems: rename is atomic, and a copy (then remove) when it cannot. The copy + * goes to a `.partial` name first, so a copy cut off midway is not mistaken for a finished one next time. + */ +function move(source: string, target: string): void { + try { + renameSync(source, target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EXDEV") { + throw error; + } + + const partial = `${target}.partial`; + + try { + cpSync(source, partial, { recursive: true, errorOnExist: true, force: false }); + renameSync(partial, target); + } catch (failed) { + rmSync(partial, { recursive: true, force: true }); + + throw failed; + } + + rmSync(source, { recursive: true, force: true }); + } +} + +/** The process id in an old folder's lock, when that process is still Pi Pocket and still using the folder. */ +function holder(legacy: string): number | undefined { + const file = join(legacy, "harness.lock"); + + if (!existsSync(file)) { + return undefined; + } + + const pid = Number(readFileSync(file, "utf8").trim()); + + return lockHolder(pid) ? pid : undefined; +} + +/** + * Move an existing `~/.pi-pocket` into the four, for people who ran an older Pi Pocket. Entries are moved only where + * the destination is absent, so a migration cut off mid-way finishes on the next start; anything unknown is left where + * it is. A folder named with `PI_POCKET_*_DIR` that is the old one itself keeps what it has. + */ +export function migrateLegacy( + dirs: Folders, + log: (line: string) => void = () => {}, + home: string = homedir(), +): void { + const legacy = join(home, ".pi-pocket"); + + if (!existsSync(legacy)) { + return; + } + + const running = holder(legacy); + + if (running !== undefined) { + // Its database is open and being written: moving it would give this process a torn copy to open. + log( + `Pi Pocket is still running on ${tilde(legacy, home)} (pid ${running}): stop it, then start again.`, + ); + + return; + } + + let known = false; + let moved = false; + let movedNamed = false; + + for (const kind of ["config", "data", "state", "cache"] as const) { + const folder = dirs[kind]; + + // A folder named with PI_POCKET_*_DIR may be the old one itself: it keeps what is already in it. + if (resolve(folder) === resolve(legacy)) { + continue; + } + + for (const entry of LEGACY_ENTRIES[kind]) { + const source = join(legacy, entry); + + if (!existsSync(source)) { + continue; + } + + known = true; + const target = join(folder, entry); + + if (existsSync(target)) { + // Two databases are worse than one: say which one is left, rather than run on it in silence. + if (entry.startsWith("pocket.sqlite")) { + log( + `Kept ${tilde(source, home)}: ${tilde(target, home)} is already there. Delete one of them.`, + ); + } + + continue; + } + + mkdirSync(folder, { recursive: true, mode: 0o700 }); + move(source, target); + moved = true; + movedNamed ||= NAMED_BY_PATH.has(entry); + } + } + + if (!known) { + return; + } + + if (moved) { + log(`Moved Pi Pocket's data from ${tilde(legacy, home)} to ${tilde(dirs.data, home)}.`); + } + + if (movedNamed) { + // What the app stored before the move: an older message's attachments, an isolated session's folder. + log( + `Files those conversations name by ${tilde(legacy, home)} are now in ${tilde(dirs.data, home)}.`, + ); + } + + const left = readdirSync(legacy); + + if (left.length === 0) { + rmdirSync(legacy); + + return; + } + + log(`Left in ${tilde(legacy, home)}: ${left.join(", ")}`); +} diff --git a/src/server/extensions/prompt.ts b/src/server/extensions/prompt.ts index 43708b17..a2ebd4a2 100644 --- a/src/server/extensions/prompt.ts +++ b/src/server/extensions/prompt.ts @@ -37,7 +37,7 @@ function docs(dataDir: string): string { return `Pi Pocket documentation (read only when someone asks about Pi Pocket itself: how it works, changing how you behave in it, extending it, or changing its code): - Start here: ${join(folder, "index.md")} -- Pi Pocket's code is in ${code}; its data (people, settings, sessions) in ${dataDir}. Change the data through the app, never by editing its files, except extensions/ in it, where drop-in extensions go, and themes/, where the owner's themes are. +- Pi Pocket's code is in ${code}; its data (sessions, uploads, extensions, themes) in ${dataDir}. Its people, roles, and settings are in its config folder (~/.config/pi-pocket, unless XDG_CONFIG_HOME or PI_POCKET_CONFIG_DIR names another), its tunnel log in its state folder, and its browser profile and downloaded cloudflared in its cache folder. Change the data through the app, never by editing its files, except extensions/ and themes/ in the data folder, which people edit by hand. - When asked about: changing how you behave without code, such as instructions, AGENTS.md, skills, or prompt templates (docs/customizing.md); a new tool, prompt section, or hook, or a page, button, or route in the app (docs/extensions.md, an index of the guides in docs/extensions/, with working examples in docs/examples/ and the exact types in types/pocket.d.ts); how the app looks or is laid out: a theme (docs/extensions/themes.md, with every token and part in docs/extensions/reference/theme-surface.md); changing Pi Pocket's own code while it runs (docs/self-editing.md); where its code is (docs/map.md); how its parts depend on each other (docs/architecture.md); what a feature does (docs/features.md) - Resolve docs/... and types/... under ${code}, not the working directory. Read a doc completely before changing anything. docs/index.md and docs/extensions.md are indexes: follow the links your task needs, not all of them. - Pi's own documentation (skills, prompt templates, models, providers, settings): ${getDocsPath()}`; diff --git a/src/server/gallery.ts b/src/server/gallery.ts index 197d854e..aad210a3 100644 --- a/src/server/gallery.ts +++ b/src/server/gallery.ts @@ -1153,7 +1153,7 @@ export class Gallery { `- browser module: ${d.client === null ? "none" : `for ${d.client}`}; widest access: ${d.access}`, "", "Read every file, and report:", - "1. What it reads and writes: files and folders, sessions and documents. Does it read config.json or anything else in ~/.pi-pocket, Pi's auth.json or ~/.pi/agent, environment variables, or other secrets?", + "1. What it reads and writes: files and folders, sessions and documents. Does it read config.json or anything else in Pi Pocket's folders (~/.config/pi-pocket, ~/.local/share/pi-pocket, ~/.local/state/pi-pocket, ~/.cache/pi-pocket), Pi's auth.json or ~/.pi/agent, environment variables, or other secrets?", "2. Its network calls: to which addresses, and what it sends.", "3. Processes it starts, and code it loads while it runs (eval, new Function, dynamic imports, downloads).", "4. What its browser module does in the app's page: its requests, what it stores, what it shows.", diff --git a/src/server/lock.ts b/src/server/lock.ts index 0c51628d..9489dced 100644 --- a/src/server/lock.ts +++ b/src/server/lock.ts @@ -1,11 +1,11 @@ -/** One Pi Pocket process per data directory: a lock file in it holds the process id of the one running there. */ +/** One Pi Pocket process per data folder: a lock file in it holds the process id of the one running there. */ import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; /** * Whether a lock file's process is still running Pi Pocket. A process id can be reused after a crash, so on Linux * (and Android) a live process must also be Node; where that cannot be read, a live process counts. */ -function lockHolder(pid: number): boolean { +export function lockHolder(pid: number): boolean { if (!Number.isInteger(pid) || pid <= 0 || pid === process.pid) { return false; } @@ -25,7 +25,7 @@ function lockHolder(pid: number): boolean { } /** - * One process per data directory. A lock left by a process that died (killed, or its phone stopped it) is taken + * One process per data folder. A lock left by a process that died (killed, or its phone stopped it) is taken * over; created with `wx`, so of two processes starting at once only one gets it. */ export function takeLock(lockFile: string): void { @@ -34,7 +34,7 @@ export function takeLock(lockFile: string): void { if (lockHolder(pid)) { throw new Error( - `Pi Pocket is already running on this data directory (pid ${pid}). If it is not, delete ${lockFile}.`, + `Pi Pocket is already running on this data folder (pid ${pid}). If it is not, delete ${lockFile}.`, ); } @@ -49,7 +49,7 @@ export function takeLock(lockFile: string): void { } throw new Error( - "Pi Pocket is already running on this data directory: another one started at the same moment.", + "Pi Pocket is already running on this data folder: another one started at the same moment.", ); } } diff --git a/src/server/main.ts b/src/server/main.ts index c37b3127..4d27b1f8 100644 --- a/src/server/main.ts +++ b/src/server/main.ts @@ -2,18 +2,18 @@ /** * Pi Pocket server. Run it through `bin/pi-pocket.js`, which restarts it when the app asks to (exit code 75). * - * node bin/pi-pocket.js [--host 127.0.0.1] [--port 8787] [--cwd DIR] [--data DIR] [--rotate-token] + * node bin/pi-pocket.js [--host 127.0.0.1] [--port 8787] [--cwd DIR] [--rotate-token] * * Under the launcher (PI_POCKET_LAUNCHER=1 with an IPC channel) the server reports `ready` instead of printing its * sign-in links, and takes `access` messages that say how other devices reach it (a tunnel's public URL). */ -import { chmodSync, mkdirSync } from "node:fs"; import { createServer } from "node:http"; import { networkInterfaces } from "node:os"; import { join, resolve } from "node:path"; // Only types from the app here: its modules load below, after the signal handlers (loading takes a moment). import type { AccessInfo, PocketApp } from "./app.ts"; -import { APP_ROOT, dataDir } from "./config.ts"; +import { APP_ROOT } from "./config.ts"; +import { ensureFolders, foldersFromEnv, migrateLegacy } from "./dirs.ts"; import type { createHandler } from "./http.ts"; const RESTART_CODE = 75; @@ -29,7 +29,6 @@ function parseArgs(argv: string[]) { host: process.env.PI_POCKET_HOST ?? "127.0.0.1", port: Number(process.env.PI_POCKET_PORT ?? 8787), cwd: process.cwd(), - data: dataDir(), rotateToken: false, }; @@ -52,14 +51,10 @@ function parseArgs(argv: string[]) { options.port = Number(value()); } else if (arg === "--cwd") { options.cwd = resolve(value()); - } else if (arg === "--data") { - options.data = resolve(value()); } else if (arg === "--rotate-token") { options.rotateToken = true; } else if (arg === "--help" || arg === "-h") { - console.log( - "pi-pocket [--host 127.0.0.1] [--port 8787] [--cwd DIR] [--data DIR] [--rotate-token]", - ); + console.log("pi-pocket [--host 127.0.0.1] [--port 8787] [--cwd DIR] [--rotate-token]"); process.exit(0); } else { throw new Error(`Unknown argument: ${arg}`); @@ -75,11 +70,20 @@ function parseArgs(argv: string[]) { const options = parseArgs(process.argv.slice(2)); -process.env.PI_POCKET_DIR = options.data; -// The data folder holds the database, sign-in tokens, push keys, and uploads: only this user may open it, also when -// --data named a folder that already existed. -mkdirSync(options.data, { recursive: true, mode: 0o700 }); -chmodSync(options.data, 0o700); +// The folders hold the database, sign-in tokens, push keys, and uploads: only this user may open them, and an older +// `~/.pi-pocket` moves into them before anything reads them. After the options, so `--help` and a bad argument leave +// the disk as they found it. +const dirs = foldersFromEnv(); + +ensureFolders(dirs); +migrateLegacy(dirs, (line) => console.log(line)); + +if (process.env.PI_POCKET_DIR !== undefined) { + // The variable that named the one folder, before the four: say so, rather than start on empty folders. + console.log( + "PI_POCKET_DIR is no longer used. Set PI_POCKET_DATA_DIR (README.md) to keep that folder's data.", + ); +} /** How long stopping may wait for running work to stop. What is still running then resumes in the next process. */ const STOP_LIMIT_MS = 10_000; @@ -189,7 +193,7 @@ const [{ PocketApp: App }, { createHandler: makeHandler }, { watchTree }] = awai import("./reload.ts"), ]); const opened = await App.open({ - dataDir: options.data, + dirs, defaultCwd: options.cwd, supervised: process.env.PI_POCKET_SUPERVISED === "1", }); @@ -244,7 +248,10 @@ if (launcher) { } console.log("\nPi Pocket is running."); - console.log(` data: ${options.data}`); + console.log(` config: ${dirs.config}`); + console.log(` data: ${dirs.data}`); + console.log(` state: ${dirs.state}`); + console.log(` cache: ${dirs.cache}`); console.log(` sessions: new sessions start in ${options.cwd}`); console.log( " Open one of these once per browser to sign in as the owner (keep them private):", diff --git a/src/server/workspace.ts b/src/server/workspace.ts index b063655f..ee220de7 100644 --- a/src/server/workspace.ts +++ b/src/server/workspace.ts @@ -285,8 +285,9 @@ export class Workspace { /** * A file a person may read whole through a session: the viewer, and files sent along with a message. As - * `conversationFile`, but Pi's own folder (sign-ins, settings) and this app's data (people, tokens, the database) - * are the owner's alone. Uploads, worktrees, and PiP's clones (`repos/`), which sessions use, are not kept back. + * `conversationFile`, but Pi's own folder (sign-ins, settings) and this app's own folders (people and tokens, the + * database, its lock and tunnel log, the browser's profile) are the owner's alone. Uploads, worktrees, and PiP's + * clones (`repos/`), which sessions use, are not kept back. */ readableFile(user: User, id: ConversationId, path: string): string { const file = this.conversationFile(user, id, path); @@ -306,9 +307,17 @@ export class Workspace { const target = real(file); const inside = (root: string) => target === root || target.startsWith(root + sep); const data = real(this.#app.dataDir); + // The old folder too: a migration leaves behind what another folder already had, `config.json` included. + const appFolders = [ + this.#app.configDir, + this.#app.stateDir, + this.#app.cacheDir, + join(this.#app.homeFolder, ".pi-pocket"), + ].map(real); if ( inside(real(getAgentDir())) || + appFolders.some(inside) || (inside(data) && !inside(join(data, "uploads")) && !inside(join(data, "worktrees")) && diff --git a/test/app.test.ts b/test/app.test.ts index 15065b90..bdcfddb9 100644 --- a/test/app.test.ts +++ b/test/app.test.ts @@ -954,7 +954,7 @@ test("a lock left by a process that is gone, or is not Node, is taken over; a ru writeFileSync(lock, `${node.pid}\n`); await assert.rejects( openHere(), - new RegExp(`already running on this data directory \\(pid ${node.pid}\\)`), + new RegExp(`already running on this data folder \\(pid ${node.pid}\\)`), ); } finally { other.kill(); diff --git a/test/browser.test.ts b/test/browser.test.ts index 5c5f8f70..8abb85ab 100644 --- a/test/browser.test.ts +++ b/test/browser.test.ts @@ -133,7 +133,7 @@ test( async () => { const saved = new Map(); const browsers = new Browsers({ - dataDir: mkdtempSync(join(root, "engine-")), + cacheDir: mkdtempSync(join(root, "engine-")), load: async (id) => saved.get(id), save: (id, value) => saved.set(id, value), }); @@ -180,7 +180,7 @@ test( test("a command sent while a navigation swaps the page waits for it", real, async () => { const browsers = new Browsers({ - dataDir: mkdtempSync(join(root, "swap-")), + cacheDir: mkdtempSync(join(root, "swap-")), load: async () => undefined, save: () => {}, }); diff --git a/test/composer-focus-ui.test.ts b/test/composer-focus-ui.test.ts index 45b15711..15a2d225 100644 --- a/test/composer-focus-ui.test.ts +++ b/test/composer-focus-ui.test.ts @@ -89,7 +89,7 @@ before(async () => { await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); base = `http://127.0.0.1:${(server.address() as { port: number }).port}`; browsers = new Browsers({ - dataDir: mkdtempSync(join(root, "composer-focus-ui-")), + cacheDir: mkdtempSync(join(root, "composer-focus-ui-")), load: async () => undefined, save: () => {}, // A mouse, whatever this machine has; the phone size's touch emulation still makes it a touch screen. diff --git a/test/dirs.test.ts b/test/dirs.test.ts new file mode 100644 index 00000000..05f62ac7 --- /dev/null +++ b/test/dirs.test.ts @@ -0,0 +1,305 @@ +// Pi Pocket's folders: where each part of its state lives, per the XDG base directory spec, and how an older +// `~/.pi-pocket` moves into them. +import assert from "node:assert/strict"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, test } from "node:test"; +import { ensureFolders, foldersFromEnv, migrateLegacy } from "../src/server/dirs.ts"; + +const root = mkdtempSync(join(tmpdir(), "pp-dirs-")); + +after(() => rmSync(root, { recursive: true, force: true })); + +/** A home folder of its own, empty, for one test. */ +function home(name: string): string { + const folder = join(root, name); + + mkdirSync(folder, { recursive: true }); + + return folder; +} + +test("the folders come from XDG_*_HOME, and from the home folder's defaults when it is unset", () => { + const xdg = foldersFromEnv( + { + XDG_CONFIG_HOME: "/xdg/config", + XDG_DATA_HOME: "/xdg/data", + XDG_STATE_HOME: "/xdg/state", + XDG_CACHE_HOME: "/xdg/cache", + }, + "/home/me", + ); + + assert.deepEqual(xdg, { + config: "/xdg/config/pi-pocket", + data: "/xdg/data/pi-pocket", + state: "/xdg/state/pi-pocket", + cache: "/xdg/cache/pi-pocket", + }); + + assert.deepEqual(foldersFromEnv({}, "/home/me"), { + config: "/home/me/.config/pi-pocket", + data: "/home/me/.local/share/pi-pocket", + state: "/home/me/.local/state/pi-pocket", + cache: "/home/me/.cache/pi-pocket", + }); +}); + +test("a relative XDG_*_HOME is ignored, as the spec says", () => { + const dirs = foldersFromEnv( + { + XDG_CONFIG_HOME: "config", + XDG_DATA_HOME: "data", + XDG_STATE_HOME: "state", + XDG_CACHE_HOME: "cache", + }, + "/home/me", + ); + + assert.deepEqual(dirs, { + config: "/home/me/.config/pi-pocket", + data: "/home/me/.local/share/pi-pocket", + state: "/home/me/.local/state/pi-pocket", + cache: "/home/me/.cache/pi-pocket", + }); +}); + +test("each PI_POCKET_*_DIR wins over the XDG folders", () => { + const dirs = foldersFromEnv( + { + XDG_CONFIG_HOME: "/xdg/config", + XDG_DATA_HOME: "/xdg/data", + XDG_STATE_HOME: "/xdg/state", + XDG_CACHE_HOME: "/xdg/cache", + PI_POCKET_CONFIG_DIR: "/elsewhere/config", + PI_POCKET_STATE_DIR: "/elsewhere/state", + }, + "/home/me", + ); + + assert.equal(dirs.config, "/elsewhere/config"); + assert.equal(dirs.data, "/xdg/data/pi-pocket"); + assert.equal(dirs.state, "/elsewhere/state"); + assert.equal(dirs.cache, "/xdg/cache/pi-pocket"); +}); + +test("an empty PI_POCKET_*_DIR falls back to the XDG folder", () => { + const dirs = foldersFromEnv({ PI_POCKET_CONFIG_DIR: "", PI_POCKET_DATA_DIR: "" }, "/home/me"); + + assert.equal(dirs.config, "/home/me/.config/pi-pocket"); + assert.equal(dirs.data, "/home/me/.local/share/pi-pocket"); +}); + +test("ensureFolders makes the four, private", () => { + const dirs = foldersFromEnv( + { + PI_POCKET_CONFIG_DIR: join(root, "made", "config"), + PI_POCKET_DATA_DIR: join(root, "made", "data"), + PI_POCKET_STATE_DIR: join(root, "made", "state"), + PI_POCKET_CACHE_DIR: join(root, "made", "cache"), + }, + home("ensure"), + ); + + ensureFolders(dirs); + + for (const folder of Object.values(dirs)) { + assert.ok(existsSync(folder), folder); + + if (process.platform !== "win32") { + assert.equal(statSync(folder).mode & 0o777, 0o700, folder); + } + } +}); + +test("an old ~/.pi-pocket moves into the four folders, and what is not Pi Pocket's stays behind", () => { + const folder = home("migrate"); + const legacy = join(folder, ".pi-pocket"); + const dirs = { + config: join(folder, "config"), + data: join(folder, "data"), + state: join(folder, "state"), + cache: join(folder, "cache"), + }; + + mkdirSync(join(legacy, "extensions"), { recursive: true }); + mkdirSync(join(legacy, "bin"), { recursive: true }); + writeFileSync(join(legacy, "config.json"), "{}"); + writeFileSync(join(legacy, "launcher.json"), "{}"); + writeFileSync(join(legacy, "pocket.sqlite"), "sqlite"); + writeFileSync(join(legacy, "pocket.sqlite-wal"), "wal"); + writeFileSync(join(legacy, "pocket.sqlite-shm"), "shm"); + writeFileSync(join(legacy, "push.json"), "{}"); + writeFileSync(join(legacy, "extensions", "mine.ts"), "/** Mine. */\n"); + writeFileSync(join(legacy, "harness.lock"), "1\n"); + mkdirSync(join(legacy, "repos", "owner", "project"), { recursive: true }); + writeFileSync(join(legacy, "repos", "owner", "project", "README.md"), "clone"); + writeFileSync(join(legacy, "cloudflared.log"), "log\n"); + writeFileSync(join(legacy, "bin", "cloudflared"), "binary"); + mkdirSync(join(legacy, "browser", "profile"), { recursive: true }); + writeFileSync(join(legacy, "browser", "profile", "Cookies"), "cookie"); + // Not a name Pi Pocket knows: it may be someone else's, so it is left where it is. + writeFileSync(join(legacy, "notes.txt"), "hello"); + + const said: string[] = []; + + migrateLegacy(dirs, (line) => said.push(line), folder); + + for (const moved of [ + join(dirs.config, "config.json"), + join(dirs.config, "launcher.json"), + join(dirs.data, "pocket.sqlite"), + join(dirs.data, "pocket.sqlite-wal"), + join(dirs.data, "pocket.sqlite-shm"), + join(dirs.data, "repos", "owner", "project", "README.md"), + join(dirs.data, "push.json"), + join(dirs.data, "extensions", "mine.ts"), + join(dirs.data, "harness.lock"), + join(dirs.state, "cloudflared.log"), + join(dirs.cache, "bin", "cloudflared"), + join(dirs.cache, "browser", "profile", "Cookies"), + ]) { + assert.ok(existsSync(moved), moved); + } + + assert.deepEqual(readdirSync(legacy), ["notes.txt"]); + assert.ok( + said.some((line) => line.includes("Moved Pi Pocket's data")), + said.join("\n"), + ); + assert.ok( + said.some((line) => line === "Left in ~/.pi-pocket: notes.txt"), + said.join("\n"), + ); + assert.ok( + said.some((line) => line.includes("Files those conversations name")), + said.join("\n"), + ); +}); + +test("migrating again does nothing, and never overwrites what is already in a folder", () => { + const folder = home("again"); + const legacy = join(folder, ".pi-pocket"); + const dirs = { + config: join(folder, "config"), + data: join(folder, "data"), + state: join(folder, "state"), + cache: join(folder, "cache"), + }; + + mkdirSync(legacy, { recursive: true }); + mkdirSync(dirs.config, { recursive: true }); + writeFileSync(join(legacy, "config.json"), "the old one"); + // The destination came first: the old one stays where it is, and the new one is not replaced. + writeFileSync(join(dirs.config, "config.json"), "the one in use"); + writeFileSync(join(legacy, "push.json"), "{}"); + + migrateLegacy(dirs, () => {}, folder); + + assert.equal(existsSync(join(dirs.data, "push.json")), true); + assert.equal(existsSync(join(legacy, "config.json")), true); + assert.equal(readdirSync(join(dirs.config)).length, 1); + + // Everything is gone now, so a second run finds nothing to do and says nothing. + rmSync(join(legacy, "config.json")); + rmSync(legacy, { recursive: true, force: true }); + + const said: string[] = []; + + migrateLegacy(dirs, (line) => said.push(line), folder); + + assert.deepEqual(said, []); + assert.deepEqual(readdirSync(dirs.data), ["push.json"]); +}); + +test("a folder named with PI_POCKET_*_DIR keeps what is already in the old one", () => { + const folder = home("named"); + const legacy = join(folder, ".pi-pocket"); + const dirs = { + config: join(folder, "config"), + data: legacy, + state: join(folder, "state"), + cache: join(folder, "cache"), + }; + + mkdirSync(legacy, { recursive: true }); + writeFileSync(join(legacy, "pocket.sqlite"), "sqlite"); + + migrateLegacy(dirs, () => {}, folder); + + assert.equal(existsSync(join(legacy, "pocket.sqlite")), true); +}); + +test("a database already in the data folder is not replaced, and the old one is named", () => { + const folder = home("two-databases"); + const legacy = join(folder, ".pi-pocket"); + const dirs = { + config: join(folder, "config"), + data: join(folder, "data"), + state: join(folder, "state"), + cache: join(folder, "cache"), + }; + + mkdirSync(legacy, { recursive: true }); + mkdirSync(dirs.data, { recursive: true }); + writeFileSync(join(legacy, "pocket.sqlite"), "the old one"); + writeFileSync(join(dirs.data, "pocket.sqlite"), "the one in use"); + + const said: string[] = []; + + migrateLegacy(dirs, (line) => said.push(line), folder); + + assert.equal(readFileSync(join(dirs.data, "pocket.sqlite"), "utf8"), "the one in use"); + assert.ok( + said.some((line) => line.includes("Kept ~/.pi-pocket/pocket.sqlite")), + said.join("\n"), + ); +}); + +test( + "a Pi Pocket still running on the old folder is not migrated under it", + { skip: process.platform === "win32" }, + async () => { + const { spawn } = await import("node:child_process"); + const { once } = await import("node:events"); + const folder = home("running"); + const legacy = join(folder, ".pi-pocket"); + const dirs = { + config: join(folder, "config"), + data: join(folder, "data"), + state: join(folder, "state"), + cache: join(folder, "cache"), + }; + const running = spawn(process.execPath, ["-e", "setTimeout(() => {}, 30000)"]); + + try { + mkdirSync(legacy, { recursive: true }); + writeFileSync(join(legacy, "pocket.sqlite"), "sqlite"); + writeFileSync(join(legacy, "harness.lock"), `${running.pid}\n`); + + const said: string[] = []; + + migrateLegacy(dirs, (line) => said.push(line), folder); + + assert.equal(existsSync(join(legacy, "pocket.sqlite")), true); + assert.equal(existsSync(join(dirs.data, "pocket.sqlite")), false); + assert.ok( + said.some((line) => line.includes("is still running on ~/.pi-pocket")), + said.join("\n"), + ); + } finally { + running.kill(); + await once(running, "exit"); + } + }, +); diff --git a/test/extension-ui.test.ts b/test/extension-ui.test.ts index a287a974..6886b849 100644 --- a/test/extension-ui.test.ts +++ b/test/extension-ui.test.ts @@ -230,7 +230,7 @@ before(async () => { await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); base = `http://127.0.0.1:${(server.address() as { port: number }).port}`; browsers = new Browsers({ - dataDir: mkdtempSync(join(root, "extension-ui-")), + cacheDir: mkdtempSync(join(root, "extension-ui-")), load: async () => undefined, save: () => {}, }); diff --git a/test/extensions-hub-ui.test.ts b/test/extensions-hub-ui.test.ts index 18975c12..bbe8ecac 100644 --- a/test/extensions-hub-ui.test.ts +++ b/test/extensions-hub-ui.test.ts @@ -70,7 +70,7 @@ test( createHandler({ app, listen: { host: "127.0.0.1", port: 0 }, restart: () => {} }), ); const browsers = new Browsers({ - dataDir: mkdtempSync(join(root, "extensions-hub-ui-")), + cacheDir: mkdtempSync(join(root, "extensions-hub-ui-")), load: async () => undefined, save: () => {}, }); diff --git a/test/helpers.ts b/test/helpers.ts index 7e0ba8b5..9e467cae 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -2,13 +2,14 @@ // this before anything from src/: it points Pi's config at a temporary folder first. import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; import type { FauxResponseStep } from "@earendil-works/pi-ai"; import { fauxAssistantMessage, fauxProvider, fauxText } from "@earendil-works/pi-ai/providers/faux"; import { type ConversationId, UsageDoc } from "@earendil-works/pi-durable"; import type { Attachment } from "../src/server/commands.ts"; import type { User } from "../src/server/config.ts"; +import type { Folders } from "../src/server/dirs.ts"; import type { Client } from "../src/server/room.ts"; export const context = BACKGROUND_CONTEXT; @@ -67,6 +68,21 @@ export function scriptedModel(route: FauxResponseStep = echo) { return faux; } +/** + * The four folders an app opened on `data` uses: the data folder as given, and config, state, and cache as its + * siblings, the way `~/.local/share/pi-pocket` sits beside its own. + */ +export function foldersFor(data: string): Folders { + const parent = dirname(data); + + return { + config: join(parent, "config"), + data, + state: join(parent, "state"), + cache: join(parent, "cache"), + }; +} + /** Open the app on `dataDir` with the scripted model. Opening the same folder again is a restart; `now` moves its clock. */ export function openApp( model: ReturnType, @@ -74,7 +90,7 @@ export function openApp( now?: () => number, ): Promise { return PocketApp.open({ - dataDir, + dirs: foldersFor(dataDir), defaultCwd: work, supervised: false, log: () => {}, diff --git a/test/launcher-ui.test.ts b/test/launcher-ui.test.ts index 610829df..479d3084 100644 --- a/test/launcher-ui.test.ts +++ b/test/launcher-ui.test.ts @@ -130,7 +130,7 @@ before(async () => { await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); base = `http://127.0.0.1:${(server.address() as { port: number }).port}`; browsers = new Browsers({ - dataDir: mkdtempSync(join(root, "launcher-ui-")), + cacheDir: mkdtempSync(join(root, "launcher-ui-")), load: async () => undefined, save: () => {}, args: [ diff --git a/test/main.test.ts b/test/main.test.ts index 8d4c3a80..dc4dc07c 100644 --- a/test/main.test.ts +++ b/test/main.test.ts @@ -31,28 +31,30 @@ async function freePort(): Promise { function server( name: string, port: number, -): { proc: ChildProcess; data: string; output: () => string } { - const data = join(root, name, "data"); - const proc = spawn( - process.execPath, - [MAIN, "--port", String(port), "--data", data, "--cwd", root], - { - env: { - ...process.env, - PI_CODING_AGENT_DIR: join(root, name, "agent"), - PI_POCKET_GUARD: "off", - PI_POCKET_LAUNCHER: "", - PI_POCKET_SUPERVISED: "", - }, - stdio: ["ignore", "pipe", "pipe"], +): { proc: ChildProcess; data: string; state: string; output: () => string } { + const folder = (kind: string) => join(root, name, kind); + const data = folder("data"); + const state = folder("state"); + const proc = spawn(process.execPath, [MAIN, "--port", String(port), "--cwd", root], { + env: { + ...process.env, + PI_CODING_AGENT_DIR: join(root, name, "agent"), + PI_POCKET_GUARD: "off", + PI_POCKET_LAUNCHER: "", + PI_POCKET_SUPERVISED: "", + PI_POCKET_CONFIG_DIR: folder("config"), + PI_POCKET_DATA_DIR: data, + PI_POCKET_STATE_DIR: state, + PI_POCKET_CACHE_DIR: folder("cache"), }, - ); + stdio: ["ignore", "pipe", "pipe"], + }); let text = ""; proc.stdout!.on("data", (chunk) => (text += chunk)); proc.stderr!.on("data", (chunk) => (text += chunk)); - return { proc, data, output: () => text }; + return { proc, data, state, output: () => text }; } async function exit( diff --git a/test/mobile-ui.test.ts b/test/mobile-ui.test.ts index 15bd2ff6..22f254e3 100644 --- a/test/mobile-ui.test.ts +++ b/test/mobile-ui.test.ts @@ -139,7 +139,7 @@ before(async () => { await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); base = `http://127.0.0.1:${(server.address() as { port: number }).port}`; browsers = new Browsers({ - dataDir: mkdtempSync(join(root, "mobile-ui-")), + cacheDir: mkdtempSync(join(root, "mobile-ui-")), load: async () => undefined, save: () => {}, }); diff --git a/test/model-scope.test.ts b/test/model-scope.test.ts index 59aa2dd5..e843be10 100644 --- a/test/model-scope.test.ts +++ b/test/model-scope.test.ts @@ -1,4 +1,4 @@ -import { cleanUp, owner, root, scriptedModel, work } from "./helpers.ts"; +import { cleanUp, foldersFor, owner, root, scriptedModel, work } from "./helpers.ts"; import assert from "node:assert/strict"; import { rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -6,7 +6,7 @@ import { after, test } from "node:test"; import { PocketApp } from "../src/server/app.ts"; const app = await PocketApp.open({ - dataDir: join(root, "scope-data"), + dirs: foldersFor(join(root, "scope-data")), defaultCwd: work, supervised: false, log: () => {}, diff --git a/test/peeks-ui.test.ts b/test/peeks-ui.test.ts index a668f826..b2755274 100644 --- a/test/peeks-ui.test.ts +++ b/test/peeks-ui.test.ts @@ -154,7 +154,7 @@ before(async () => { await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); base = `http://127.0.0.1:${(server.address() as { port: number }).port}`; browsers = new Browsers({ - dataDir: mkdtempSync(join(root, "peeks-ui-")), + cacheDir: mkdtempSync(join(root, "peeks-ui-")), load: async () => undefined, save: () => {}, }); diff --git a/test/pip-chat-ui.test.ts b/test/pip-chat-ui.test.ts index 2014d582..a6694e32 100644 --- a/test/pip-chat-ui.test.ts +++ b/test/pip-chat-ui.test.ts @@ -252,7 +252,7 @@ before(async () => { await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); base = `http://127.0.0.1:${(server.address() as { port: number }).port}`; browsers = new Browsers({ - dataDir: mkdtempSync(join(root, "pip-chat-browser-")), + cacheDir: mkdtempSync(join(root, "pip-chat-browser-")), load: async () => undefined, save: () => {}, args: [ diff --git a/test/pip-models.test.ts b/test/pip-models.test.ts index 93997bfa..8d4c3a02 100644 --- a/test/pip-models.test.ts +++ b/test/pip-models.test.ts @@ -1,6 +1,7 @@ import { type App, cleanUp, + foldersFor, home, lastText, newSession, @@ -82,7 +83,7 @@ let app: App; before(async () => { app = await PocketApp.open({ - dataDir: join(root, "pip-models-data"), + dirs: foldersFor(join(root, "pip-models-data")), defaultCwd: work, supervised: false, log: () => {}, diff --git a/test/qol.test.ts b/test/qol.test.ts index c43d6ded..6ba94b07 100644 --- a/test/qol.test.ts +++ b/test/qol.test.ts @@ -17,7 +17,7 @@ import { import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { after, before, test } from "node:test"; import type { FauxResponseStep } from "@earendil-works/pi-ai"; import { fauxAssistantMessage, fauxText } from "@earendil-works/pi-ai/providers/faux"; @@ -271,7 +271,7 @@ test("after a restart, an older message settling does not take Pi back from the } }); -test("only the owner may read Pi's folder and the app's data, but anyone who steers may read uploads", async () => { +test("only the owner may read Pi's folder and Pi Pocket's own folders, but anyone who steers may read uploads", async () => { const id = await newSession(app, folder); const { user: guest } = app.config.addUser("Guest", "guest"); @@ -280,10 +280,28 @@ test("only the owner may read Pi's folder and the app's data, but anyone who ste writeFileSync(agentFile, "{}"); await assert.rejects(app.workspace.viewFile(id, guest, agentFile), /is not there/); - await assert.rejects( - app.workspace.viewFile(id, guest, join(app.dataDir, "config.json")), - /is not there/, - ); + + // Everything the folders hold is the owner's alone: tokens, the database, push keys, the tunnel log, + // the browser's cookies, and what an unfinished migration left in the old folder. + const secret = (path: string) => { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, "{}"); + + return path; + }; + + for (const path of [ + join(app.configDir, "config.json"), + join(app.configDir, "launcher.json"), + join(app.dataDir, "pocket.sqlite"), + join(app.dataDir, "push.json"), + join(app.stateDir, "cloudflared.log"), + join(app.cacheDir, "browser", "profile", "Cookies"), + join(app.homeFolder, ".pi-pocket", "config.json"), + ]) { + await assert.rejects(app.workspace.viewFile(id, guest, secret(path)), /is not there/); + } + assert.equal((await app.workspace.viewFile(id, owner(app), agentFile)).kind, "text"); mkdirSync(app.workspace.uploadDirectory(id), { recursive: true }); writeFileSync(join(app.workspace.uploadDirectory(id), "shared.txt"), "hi"); diff --git a/test/session-row-ui.test.ts b/test/session-row-ui.test.ts index ed78d9f9..94cfe369 100644 --- a/test/session-row-ui.test.ts +++ b/test/session-row-ui.test.ts @@ -90,7 +90,7 @@ before(async () => { await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); base = `http://127.0.0.1:${(server.address() as { port: number }).port}`; browsers = new Browsers({ - dataDir: mkdtempSync(join(root, "session-row-ui-")), + cacheDir: mkdtempSync(join(root, "session-row-ui-")), load: async () => undefined, save: () => {}, // A mouse, whatever this machine has: headless Chromium without one says (hover: none), and rows show their diff --git a/test/sign-in-ui.test.ts b/test/sign-in-ui.test.ts index 4bbe63b1..1723d953 100644 --- a/test/sign-in-ui.test.ts +++ b/test/sign-in-ui.test.ts @@ -1,4 +1,13 @@ -import { type App, cleanUp, home, root, scriptedModel, until, work } from "./helpers.ts"; +import { + type App, + cleanUp, + foldersFor, + home, + root, + scriptedModel, + until, + work, +} from "./helpers.ts"; import assert from "node:assert/strict"; import { mkdtempSync } from "node:fs"; import { createServer, type Server } from "node:http"; @@ -171,7 +180,7 @@ before(async () => { const faux = scriptedModel(); app = await PocketApp.open({ - dataDir: join(root, "sign-in-ui-data"), + dirs: foldersFor(join(root, "sign-in-ui-data")), defaultCwd: work, supervised: false, log: () => {}, @@ -187,7 +196,7 @@ before(async () => { await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); base = `http://127.0.0.1:${(server.address() as { port: number }).port}`; browsers = new Browsers({ - dataDir: mkdtempSync(join(root, "sign-in-ui-browser-")), + cacheDir: mkdtempSync(join(root, "sign-in-ui-browser-")), load: async () => undefined, save: () => {}, }); diff --git a/test/signins.test.ts b/test/signins.test.ts index 7f9c831f..eafa12f5 100644 --- a/test/signins.test.ts +++ b/test/signins.test.ts @@ -2,6 +2,7 @@ import { type App, cleanUp, fakeTab, + foldersFor, home, owner, root, @@ -73,7 +74,7 @@ function expiredSignIn(): void { before(async () => { expiredSignIn(); app = await PocketApp.open({ - dataDir: join(root, "signins-data"), + dirs: foldersFor(join(root, "signins-data")), defaultCwd: work, supervised: false, log: () => {}, diff --git a/test/subagents-ui.test.ts b/test/subagents-ui.test.ts index 070d1de1..761fc465 100644 --- a/test/subagents-ui.test.ts +++ b/test/subagents-ui.test.ts @@ -132,7 +132,7 @@ before(async () => { await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); base = `http://127.0.0.1:${(server.address() as { port: number }).port}`; browsers = new Browsers({ - dataDir: mkdtempSync(join(root, "subagents-ui-")), + cacheDir: mkdtempSync(join(root, "subagents-ui-")), load: async () => undefined, save: () => {}, });