diff --git a/FEATURES.md b/FEATURES.md index b1fb5d5b..ff371cfc 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -4,13 +4,13 @@ Every row is one behaviour from the clean-room spec. Status per ID: **spec’d** (every row) → **built** → **tested** (a unit or e2e test names the ID) → **feels right** (lead sign-off after a hands-on review against `feel.md`). -**Totals:** 472 spec’d · 218 built (75 partial) · 302 tested · 0 feel right. 13 more are out of scope and not counted. +**Totals:** 472 spec’d · 227 built (80 partial) · 316 tested · 0 feel right. 13 more are out of scope and not counted. | Priority | Spec’d | Built | Tested | |---|---|---|---| | P0 | 174 | 135 | 161 | -| P1 | 219 | 71 | 127 | -| P2 | 79 | 12 | 14 | +| P1 | 219 | 77 | 138 | +| P2 | 79 | 15 | 17 | P0 = milestone-1 core experience, P1 = parity, P2 = later. @@ -338,7 +338,7 @@ Spec: `spec/archive.md` | AR-08 | Retention of archived entries | P1 | ✅ | ✅ e2e+unit | – | settings.archiveKeepDays, default 30; 30 Days / 90 Days / 1 Year / Forever from the Archive entry’s menu in the Library and Archive → Keep Archived Tabs For (macOS) until Settings exist. A shorter time that would delete entries asks first, in the Library, saying how many (Cancel keeps them all and the time as it was). Purged once the Archive is read at launch, hourly, and at once when a shorter time is confirmed; browsing history is not touched | | AR-09 | Reopen recently closed tabs | P0 | ✅ | ✅ e2e+unit | – | ⌘⇧T and ⌘Z walk back through closes with full history (TB-07, TB-18); ⌘Z also shows the tab again when it was the one shown; a closed Peek comes back through the same key (PK-10, peek lane). A page closed in the last 10 minutes keeps its history rank in the command bar (as the archived row). macOS File menu: Reopen Closed Tab and a Recently Closed submenu of the last 10; on every platform the Archive toolbar has a Recently Closed menu. Neither list shows a tab of a private or deleted profile | | AR-10 | Browsing history view | P1 | 🟡 partial | ✅ e2e+unit | – | History stays a Library section until Orbit has internal pages (D39), not a page in a tab. ⌘Y (Ctrl+H), Archive → View History (macOS) and the command bar open it next to the Archive: every visit of the current profile from the history store, newest first by day, virtualised and paged, with search (all words, title or address). A row opens its page in a new tab (⌘-click or middle-click: behind the Library); its × or its menu (Open in New Tab, Open in Background, Copy Link, Remove from History) removes every visit to the page. Clearing the Archive leaves it alone | -| AR-11 | Sidebar backups and "Restore Data" | P2 | ✅ | ✅ e2e+unit | – | Backups of the whole sidebar (Spaces, each profile’s Favorites, pinned and Today tabs, folders, split views; no private profiles) in userData/backups, one JSON file each: taken within 5 minutes of a sidebar change (only when it differs from the newest; browsing alone never makes one), once 30 s after launch, before every restore, and with Back Up Now. Kept: the newest 10 of today, then the newest of each of the 10 days before, of each week of the month before that and of each 30 days of the year before that; a deleted profile’s Favorites, Spaces and tabs are taken out of every backup. Help → Restore Data… (macOS) and the command bar open the Backups section of the Library: a dated list (day headings, time, Spaces and tabs; a file that can’t be read is listed as such, without Restore); Restore… asks first, then puts the sidebar back: tabs in both keep their page, tabs only in the backup come back asleep (with the page and history the Archive kept for them, leaving the Archive), tabs opened since go to the Archive, Spaces added since go; private windows and their tabs are left alone; Favorites over 12 go to Today (D33). When the sidebar can’t be backed up first, nothing changes. Each restore is one step on the sidebar’s undo stack (TB-18): the toast’s Undo or ⌘Z, also in the Library, puts back the sidebar it replaced, a second ⌘Z goes further back and ⌘⇧Z restores again; earlier steps stay. After an undo, a window moved off a removed Space stays where it went, and the pages come back asleep | +| AR-11 | Sidebar backups and "Restore Data" | P2 | ✅ | ✅ e2e+unit | – | Backups of the whole sidebar (Spaces, each profile’s Favorites, pinned and Today tabs, folders, split views; no private profiles) in userData/backups, one JSON file each: taken within 5 minutes of a sidebar change (only when it differs from the newest; browsing alone never makes one), once 30 s after launch, before every restore, and with Back Up Now. Kept: the newest 10 of today, then the newest of each of the 10 days before, of each week of the month before that and of each 30 days of the year before that; a deleted profile’s Favorites, Spaces and tabs are taken out of every backup. Help → Restore Data… (macOS) and the command bar open the Backups section of the Library: a dated list (day headings, time, Spaces and tabs; a file that can’t be read is listed as such, without Restore); Restore… asks first, then puts the sidebar back: tabs in both keep their page, tabs only in the backup come back asleep (with the page and history the Archive kept for them, leaving the Archive), tabs opened since go to the Archive, Spaces added since go; private windows and their tabs are left alone; Favorites over 12 go to Today (D33). When the sidebar can’t be backed up first, nothing changes. A running import stops first: none of the tabs it hasn’t shown yet go to the Archive, and its window says why (D65). Each restore is one step on the sidebar’s undo stack (TB-18): the toast’s Undo or ⌘Z, also in the Library, puts back the sidebar it replaced, a second ⌘Z goes further back and ⌘⇧Z restores again; earlier steps stay. After an undo, a window moved off a removed Space stays where it went, and the pages come back asleep | ## Keyboard shortcuts and pointer bindings (prefix KS) @@ -537,7 +537,7 @@ Spec: `spec/settings.md` |---|---|---|---|---|---|---| | ST-01 | Settings window | P0 | ✅ | ✅ e2e+unit | – | own window (⌘, / Ctrl+,): opens or focuses one; remembers section and size; takes the opener Space's colours; sections come from a registry (src/renderer/src/settings/registry.ts); search filters rows | | ST-02 | Settings reachable and searchable from the command bar | P1 | ✅ | ✅ e2e | – | sections and common settings are command-bar actions (settings.*) that open Settings at the row; the keys lane adds Shortcuts | -| ST-03 | General section | P1 | 🟡 partial | ✅ e2e | – | default browser row, app-menu item and command-bar action (only an installed build can register; e2e uses a stub); Mini Window archive timing; Import button once the import lane registers import.open; Updates group: automatic checks switch, Check Now and the update feed address. Not built: UP-04's install-while-idle, hover-preview toggles (OT-26, P2), Open PDFs (P2) | +| ST-03 | General section | P1 | 🟡 partial | ✅ e2e | – | default browser row, app-menu item and command-bar action (only an installed build can register; e2e uses a stub); Mini Window archive timing; Import… opens the import flow (IM-01); Updates group: automatic checks switch, Check Now and the update feed address. Not built: UP-04's install-while-idle, hover-preview toggles (OT-26, P2), Open PDFs (P2) | | ST-04 | Profiles section (list, rename, delete) | P1 | ✅ | ✅ e2e | – | list with Default first, rename any profile, New Profile…, delete with confirmation (off while a Space uses it, and says why) | | ST-05 | Per-profile: auto-archive timing | P1 | ✅ | ✅ e2e+unit | – | 12 hours (default), 24 hours, 7 days, 30 days per profile; no Never; other profiles follow Default until set | | ST-06 | Per-profile: default search engine and custom engines | P0 | ✅ | ✅ e2e+unit | – | per-profile default engine (8 built-ins), custom engines with keyword and %s URL, Make default from the row menu; no extension override in Electron, so that row does not exist | @@ -563,23 +563,23 @@ Spec: `spec/import.md` | ID | Feature | P | Built | Tested | Feels right | Notes | |---|---|---|---|---|---|---| -| IM-01 | Import entry points | P1 | – | ✅ e2e | – | | -| IM-02 | Choose the source browser and profile(s) | P1 | – | – | – | | -| IM-03 | Choose which data types to import | P1 | – | – | – | | -| IM-04 | Choose which bookmarks to import | P1 | – | – | – | | -| IM-05 | Mapping bookmarks into Orbit (pinned tabs and folders) | P1 | – | – | – | | -| IM-06 | Target Space and profile for imported data | P1 | – | – | – | | -| IM-07 | History import | P1 | – | – | – | | +| IM-01 | Import entry points | P1 | 🟡 partial | ✅ e2e | – | the import window (a child of the browser window, in its Space's colours): Back / Next / Cancel, Cancel at any step imports nothing, Enter is Next and Escape is Back, ⌘W closes it. Opened from onboarding (the same flow inline, OB-03), the command bar's Import from Another Browser, Settings → General → Import…, and on macOS the app menu's Import from Another Browser… (macOS-unverified). Not built: a ☰ entry on Windows/Linux (Orbit has no ☰ menu). In the progress view, Cancel is offered only while the source is read; once adding starts it offers Hide, and the bookmarks added are one undo step. The window isn’t modal: the browser window stays usable while it’s open, as IM-14 asks | +| IM-02 | Choose the source browser and profile(s) | P1 | 🟡 partial | ✅ e2e+unit | – | the browsers found on this computer (Chrome, Chromium, Brave, Edge, Vivaldi, Opera, Opera GX, Firefox; Safari on macOS), each a tile in its colour with its profile or profile count; those not installed aren't shown. Several profiles: a step to choose one or more, by name. Profiles come from Chromium's Local State (or a single Default folder, or Opera's own folder) and Firefox's profiles.ini; every file is read from a copy. Safari without Full Disk Access is shown as blocked, with the way round it (export a bookmarks file from Safari and import that). Tests search only made-up fixture folders. Letter tiles, not logos: Orbit doesn't ship other browsers' marks. Safari's reader (Bookmarks.plist, a binary property list) is tested on fixtures, macOS-unverified | +| IM-03 | Choose which data types to import | P1 | 🟡 partial | ✅ e2e+unit | – | after the source and its profiles, what they hold as checkboxes, all on, with counts ("6 bookmarks, as pinned tabs", "6 pages visited"); types a source can't provide aren't shown, and a profile whose file the browser holds offers the rest; never passwords or cards (D52). Cancel (Do this later in onboarding) weighs the same as Next. Extensions are listed, off, as not importable yet (IM-09). Cookies (IM-10) aren’t offered. A file that is there but can’t be read is named with why, e.g. “Work: history couldn’t be read (the file is damaged)”, and the profile’s other types are still offered | +| IM-04 | Choose which bookmarks to import | P1 | ✅ | ✅ e2e+unit | – | the source's tree (Chromium's bookmarks bar, other and mobile bookmarks; Firefox's toolbar, menu, other and mobile bookmarks; Safari's favorites bar, menu and reading list; a file's own groups), several profiles each under its name, with a checkbox per folder and bookmark; everything chosen at first; Select all / none; "4 of 6 bookmarks selected". Addresses that can't be pinned show unticked with the reason. Folders below the top level start closed, and so does a top-level one holding over 1,000 bookmarks. The list isn’t virtualized: every row at the top level or in an open folder is drawn, so a large file shows quickly only when its bookmarks sit in such closed folders (20,000 in folders of 2,000 show in about 0.4 s). 20,000 bookmarks at the top level take about 2.9 s to show; 200,000 there, or 200 top-level folders of 1,000, about 30 s; 10,000 top-level folders about 6 s; and opening a large folder draws all its rows. Only the import window waits | +| IM-05 | Mapping bookmarks into Orbit (pinned tabs and folders) | P1 | 🟡 partial | ✅ e2e+unit | – | one pinned folder "Imported from " ("Imported bookmarks" from a file) at the end of the target Space's pinned section: the source's top-level groups and nested folders kept, in order, as closed sub-folders (the folder itself closed above 1,000 items); several profiles into one Space each get a sub-folder by name; each bookmark a pinned tab with its title and address, asleep, with no favicon or request until opened; bookmarklets, saved searches, browser-only and non-web addresses skipped and counted; Firefox tags and separators left out; no Favorites made; one undo step takes it all back. 2,000 bookmarks take about 0.3–0.4 s, made out of sight in parts and shown at once (IM-14), with the longest main-process stall about 40–50 ms. Not built: fetching a favicon when an imported row scrolls into view; its icon comes when the tab is first opened | +| IM-06 | Target Space and profile for imported data | P1 | ✅ | ✅ e2e+unit | – | the last step: the profile (it gets the history) and the Space (its pinned section gets the bookmarks): the current Space by default, the first one in onboarding, or New Space… with a name. Several source profiles: "Make a matching Orbit profile and Space for each" makes them, in order, named after the source profiles, each getting its own profile's history. Private windows' profiles aren't offered and never get history | +| IM-07 | History import | P1 | ✅ | ✅ e2e+unit | – | Chromium's History, Firefox's places.sqlite and Safari's History.db (macOS-unverified), read from copies a window of rows at a time, into the chosen profile's history: each page with its title, visit and typed counts and its newest visits (up to 20). History (⌘Y / Ctrl+H) lists them and the command bar suggests and completes them at once; no tab is made and nothing loads. Left out: browsers' own pages, hidden pages, frame loads (Chromium subframes; Firefox embeds, framed links, downloads). History keeps the 50,000 most recent pages: a larger history keeps its newest and the summary counts the rest. 100,000 pages take about 2 s, with main's longest stall about 50–110 ms and none in the windows; the profile's history is written once, as a snapshot | | IM-08 | Password import | P1 | ⛔ out of scope | – | – | Orbit has no password manager to import into: the spec has no row for one (OT-11), and Electron has none. IM-03 offers no Passwords type, and IM-12 and IM-13 take no passwords CSV (D52) | -| IM-09 | Extension import (Chromium-family sources) | P1 | – | – | – | | +| IM-09 | Extension import (Chromium-family sources) | P1 | 🟡 partial | ✅ e2e+unit | – | A Chromium-family profile's extensions are listed on the types step, from its Preferences and Secure Preferences (read from copies; a name kept only in the manifest is read from it, in its default language): pinned ones first, the browser's own left out. The row is off and says they aren't importable yet; unpacked ones never come over. Not built: installing them fresh from the store with one summary confirmation, pinning the pinned ones, and the ad-blocker de-duplication (blockers are already marked), because the extensions lane's install path isn't on main yet. Firefox add-ons aren't listed (the spec covers Chromium-family sources) | | IM-10 | Cookies / signed-in sessions import | P2 | – | – | – | | | IM-11 | Autofill addresses and payment cards import | P2 | ⛔ out of scope | – | – | Orbit has no store for addresses and cards to import into: the spec has no row for one (OT-12), and Electron has none (D52) | -| IM-12 | Bringing data over from Arc (user-facing exports only) | P1 | – | – | – | | -| IM-13 | Import from files (standard formats) | P1 | – | – | – | | -| IM-14 | Progress, summary and errors | P1 | – | – | – | | -| IM-15 | Re-import and duplicates | P2 | – | – | – | | -| IM-16 | Starter suggestions from imported history | P2 | – | – | – | | -| IM-17 | Export Orbit's pinned items (anti-lock-in) | P2 | – | – | – | | +| IM-12 | Bringing data over from a browser Orbit can’t read (user-facing exports only) | P1 | ✅ | ✅ e2e+unit | – | Paste a list of links (one address per line, "Title — address" with a dash, bar or tab, Markdown links, bullets): one pinned folder named by the user, a pinned tab per link in order, asleep; lines that aren't links are counted in the summary. ⌘V / Ctrl+V on the sidebar (outside a text field) with two or more links on the clipboard opens the same import with them filled in. A list over about 1 MB or 10,000 lines, pasted or from a file, isn’t put in the text field: the import window shows its count and first eight lines, read-only, with Clear, and imports it as it is. A paste over 64 MB as UTF-8 (22.5 million CJK characters, say) says “This is too large to import (the limit is 64 MB)”, as a file does, and none of it leaves the window it was pasted in. Clear also frees main’s copy of a held list. The other routes (a bookmarks file, notes, canvases, site styles) are explained in the import window's help, which reads nothing from another browser; passwords are out of scope (D52) | +| IM-13 | Import from files (standard formats) | P1 | ✅ | ✅ e2e+unit | – | bookmarks HTML (the Netscape format; Chromium- and Firefox-style exports tested) and a .txt list of links, from From a file… or dropped on the import window or the sidebar. Bookmarks map as IM-05 into "Imported bookmarks"; links as IM-12 (a list over about 1 MB or 10,000 lines shows only its count and first lines); a file with neither says "No bookmarks found in this file" and imports nothing; a cut-off file imports what it holds. Files are read in a worker, in time linear in their size (D58): main’s longest stall is about 0.07 s for a 16 MB file or list, and about 0.2 s for a 60 MB file of 200,000 bookmarks. A file nested deeper than 64 folders keeps its deeper bookmarks in the 64th; a file gives at most 200,000 bookmarks (folders don’t count) and 20,000 folders, the bookmarks of folders past that kept in the folder that holds them; over 64 MB, every route, a paste included, says “This is too large to import (the limit is 64 MB)” without reading it; a read that fails says “Orbit couldn’t read this file”. No passwords CSV (D52) | +| IM-14 | Progress, summary and errors | P1 | ✅ | ✅ e2e+unit | – | progress per data type (a spinner, then a check), then a summary with counts per type and the skipped items with reasons; Done shows the imported folder in the sidebar (its Space, the pinned section open, the folder in view). Bookmarks are made out of sight, in parts, and their folder is shown whole, at once (D62): while 20,000 are added, the browser window’s longest freeze is under 0.1 s; 100,000 take about 34 s, with the window’s longest freeze about 0.34 s and main’s longest stall about 0.57 s (saving the state). A file the source browser holds (it can't be copied) shows "Chrome is using some of its files. Quit Chrome and try again, or import the rest" with Try again; the types it could read can still be imported. History shows the pages read, then added; Done says they're in History and the command bar; what History can't keep is listed with the reason. The summary lists every item left out with its reason: skipped bookmarks (also in a folder that holds nothing else), those past 200,000 bookmarks in one file, list or profile (folders don’t count, so the summary and the selection count agree), lines that aren’t links, and a profile’s file that couldn’t be read (“Work: history couldn’t be read (the file is damaged)”). Until the folder shows, nothing lists its tabs (D65): not the command bar (by title, by address or among recent tabs), the recent-tab switcher or the Library. One the user reached anyway and placed stays where they put it, and the folder leaves it out. A quit or a failure part-way takes back exactly the tabs the import made, by id, wherever they are, and nothing of the user’s; a crash part-way leaves none either, as the next start removes tabs that no section shows. Restore Data part-way stops the import first: its tabs are removed without going to the Archive, and the import window says “The import stopped: data was restored from a backup.” Copies of the other browser’s files go in a folder under Orbit’s user data, removed at quit (a copy still being made too) and cleared at startup (D58) | +| IM-15 | Re-import and duplicates | P2 | ✅ | ✅ e2e+unit | – | the same browser or file again makes a new "Imported from (2)" folder (then (3)…), never merging into or changing the first. History de-duplicates silently: a page already imported gains only the visits it lacked, so its counts aren't doubled. Passwords aren't imported (D52) | +| IM-16 | Starter suggestions from imported history | P2 | ✅ | ✅ e2e+unit | – | after onboarding's import of a history, the Done step offers the six sites visited most (each by its most-visited page, or its home page when that's a search): the first three as Favorites, the rest as pinned tabs. One click takes one (another takes it back); Dismiss all drops them. They're added unloaded with the import when Start browsing ends onboarding, never with Skip setup (D44). Orbit can't tell web apps from other sites, so it ranks by visits; a suggested pinned tab is named after its site until first opened | +| IM-17 | Export Orbit's pinned items (anti-lock-in) | P2 | ✅ | ✅ e2e+unit | – | Export Pinned Items as Bookmarks… in the File menu (macOS-unverified: Orbit has no menu bar elsewhere), the command bar, and Settings → General → Import and export (with Include Today tabs) writes a standard bookmarks HTML file through the system's save dialog: the Favorites a folder (one per profile when there are several), each Space a folder of its pinned section with its folders nested as they are, a split's tabs in its place; Today's tabs only when chosen, as a Today folder in their Space. Orbit's own pages and private windows' profiles are left out. Copy All Links on a folder is the tabs lane's (it exists) | ## Onboarding (first run), tips and tutorials @@ -589,18 +589,18 @@ Spec: `spec/onboarding.md` |---|---|---|---|---|---|---| | OB-01 | Welcome screen | P1 | ✅ | ✅ e2e | – | a window of its own before any browser window on a first run (gated by settings.onboarded; e2e runs skip it unless ORBIT_ONBOARDING=1): Orbit's logo with the moon in motion (still with reduced motion), a welcome line, Get started; steps slide in (about 400 ms, none with reduced motion); Enter continues, Esc goes back; each step's title takes focus; progress dots are labelled. No sounds or music (optional), no haptics (Electron has no API), no first-window shimmer | | OB-02 | No account, no sign-in, ever | P0 | ✅ | ✅ e2e+unit | – | audit and e2e: no account UI anywhere, onboarding included; a first run sends nothing off the computer, with or without onboarding, except Chromium's spell-check dictionary download (see tests) | -| OB-03 | Import step | P1 | 🟡 partial | – | – | placeholder step that says import isn't available yet, because the import lane has no import.open; when it has, the step offers Import… (opens that flow in a browser window) and Do this later with the same weight. Not inline, no background import or counts, no link for people moving from another browser yet | +| OB-03 | Import step | P1 | 🟡 partial | ✅ e2e | – | the import flow inline (IM-01 to IM-06): the browsers on this computer, several profiles allowed, the data types and the bookmark tree, a bookmarks file dropped or chosen, the paste route and the help for people moving from another browser, with Do this later weighing the same as Next. An import chosen there is read at once and added when Start browsing ends onboarding, where the Done screen sums it up; Skip setup adds nothing (D44). Letter tiles, not browser logos (as IM-02) | | OB-04 | Pick a theme colour | P1 | ✅ | ✅ e2e | – | the Space theme picker (same ThemeEditor) for the first Space, applied live, with a preview of sidebar and frame, grain and intensity; Light / Dark / Match system (Match system preselected, Continue works at once); says it can be changed later. No haptics | | OB-05 | Pick favourite web apps (seed Favorites) | P1 | ✅ | ✅ e2e+unit | – | 16 web apps as letter tiles (no logos fetched), nothing preselected, at most 12; they become the first Space's profile's Favorites, unloaded until opened. No ordering by imported history (no import yet) | | OB-06 | Ad-blocking choice | P1 | – | – | – | there is no ad and tracker blocker (OT-15), so the step is left out of the flow | | OB-07 | Set as default browser | P1 | ✅ | ✅ e2e | – | explains what the default browser does; Make Orbit my default and Not now with the same weight; on Windows says what to pick in Default apps; status asked again when the window comes back to the front; no reminders later. Test runs use a stub | | OB-08 | Default search engine choice | P2 | ✅ | ✅ e2e | – | a Search with row on the Done screen: the Default profile's engine (ST-06's list), Google until changed | -| OB-09 | Done screen and the first browser window | P1 | 🟡 partial | ✅ e2e | – | Done screen sums up colours, Favorites, default browser and import; Start browsing (Back to browsing after a reset) opens the first window centred at about 86% of the screen with the colours and Favorites picked, command bar closed. No Orbit basics folder or local welcome tab (no local guide pages yet), no import counts | +| OB-09 | Done screen and the first browser window | P1 | 🟡 partial | ✅ e2e | – | Done screen sums up colours, Favorites, default browser and import (what will be added from where, e.g. "6 bookmarks from bookmarks.html", or that nothing was and where import lives); Start browsing (Back to browsing after a reset) opens the first window centred at about 86% of the screen with the colours and Favorites picked, command bar closed. No Orbit basics folder or local welcome tab (no local guide pages yet) | | OB-10 | Getting-started guide | P1 | – | – | – | | | OB-11 | Contextual tips and one-time hints | P1 | – | – | – | | | OB-12 | Essential keyboard shortcuts sheet | P1 | – | – | – | | | OB-13 | Help and feedback entry points | P1 | 🟡 partial | ✅ e2e+unit | – | command-bar Help, Report a Bug, What's New and Copy Build Info; macOS Help menu (Orbit Help, Report a Bug…, Release Notes, Troubleshooting → Copy Build Info). Help opens the project README; Report a Bug opens a new issue on the public tracker with the build info filled in. The shortcut sheet entry comes with OB-12 (keys lane) | -| OB-14 | Skip, resume and reset onboarding | P2 | ✅ | ✅ e2e+unit | – | Skip setup on every step but the last gives the spec's default first window: the theme step's colours and mode are undone and no Favorites are added (D44). The step and the picks so far are saved, so a quit part-way resumes there with them; Reset Onboarding (command bar only) runs the flow again over the open windows and deletes nothing | +| OB-14 | Skip, resume and reset onboarding | P2 | ✅ | ✅ e2e+unit | – | Skip setup on every step but the last gives the spec's default first window: the theme step's colours and mode are undone and no Favorites are added (D44). The step, the theme and the Favorites picked so far are saved, so a quit part-way resumes there with them. An import chosen at the import step isn’t: it waits in memory for Start browsing, so a quit before then loses it, and the Done step says nothing was imported (it’s in Settings, under General); Reset Onboarding (command bar only) runs the flow again over the open windows and deletes nothing | | OB-15 | AI features opt-in step | OOS | ⛔ out of scope | – | – | the spec leaves it out | | OB-16 | Personal welcome card | P2 | – | – | – | | diff --git a/docs/feature-status.json b/docs/feature-status.json index e3110d78..8fceaa53 100644 --- a/docs/feature-status.json +++ b/docs/feature-status.json @@ -68,7 +68,7 @@ "AR-08": {"built": "yes", "note": "settings.archiveKeepDays, default 30; 30 Days / 90 Days / 1 Year / Forever from the Archive entry’s menu in the Library and Archive → Keep Archived Tabs For (macOS) until Settings exist. A shorter time that would delete entries asks first, in the Library, saying how many (Cancel keeps them all and the time as it was). Purged once the Archive is read at launch, hourly, and at once when a shorter time is confirmed; browsing history is not touched"}, "AR-09": {"built": "yes", "note": "⌘⇧T and ⌘Z walk back through closes with full history (TB-07, TB-18); ⌘Z also shows the tab again when it was the one shown; a closed Peek comes back through the same key (PK-10, peek lane). A page closed in the last 10 minutes keeps its history rank in the command bar (as the archived row). macOS File menu: Reopen Closed Tab and a Recently Closed submenu of the last 10; on every platform the Archive toolbar has a Recently Closed menu. Neither list shows a tab of a private or deleted profile"}, "AR-10": {"built": "partial", "note": "History stays a Library section until Orbit has internal pages (D39), not a page in a tab. ⌘Y (Ctrl+H), Archive → View History (macOS) and the command bar open it next to the Archive: every visit of the current profile from the history store, newest first by day, virtualised and paged, with search (all words, title or address). A row opens its page in a new tab (⌘-click or middle-click: behind the Library); its × or its menu (Open in New Tab, Open in Background, Copy Link, Remove from History) removes every visit to the page. Clearing the Archive leaves it alone"}, - "AR-11": {"built": "yes", "note": "Backups of the whole sidebar (Spaces, each profile’s Favorites, pinned and Today tabs, folders, split views; no private profiles) in userData/backups, one JSON file each: taken within 5 minutes of a sidebar change (only when it differs from the newest; browsing alone never makes one), once 30 s after launch, before every restore, and with Back Up Now. Kept: the newest 10 of today, then the newest of each of the 10 days before, of each week of the month before that and of each 30 days of the year before that; a deleted profile’s Favorites, Spaces and tabs are taken out of every backup. Help → Restore Data… (macOS) and the command bar open the Backups section of the Library: a dated list (day headings, time, Spaces and tabs; a file that can’t be read is listed as such, without Restore); Restore… asks first, then puts the sidebar back: tabs in both keep their page, tabs only in the backup come back asleep (with the page and history the Archive kept for them, leaving the Archive), tabs opened since go to the Archive, Spaces added since go; private windows and their tabs are left alone; Favorites over 12 go to Today (D33). When the sidebar can’t be backed up first, nothing changes. Each restore is one step on the sidebar’s undo stack (TB-18): the toast’s Undo or ⌘Z, also in the Library, puts back the sidebar it replaced, a second ⌘Z goes further back and ⌘⇧Z restores again; earlier steps stay. After an undo, a window moved off a removed Space stays where it went, and the pages come back asleep"}, + "AR-11": {"built": "yes", "note": "Backups of the whole sidebar (Spaces, each profile’s Favorites, pinned and Today tabs, folders, split views; no private profiles) in userData/backups, one JSON file each: taken within 5 minutes of a sidebar change (only when it differs from the newest; browsing alone never makes one), once 30 s after launch, before every restore, and with Back Up Now. Kept: the newest 10 of today, then the newest of each of the 10 days before, of each week of the month before that and of each 30 days of the year before that; a deleted profile’s Favorites, Spaces and tabs are taken out of every backup. Help → Restore Data… (macOS) and the command bar open the Backups section of the Library: a dated list (day headings, time, Spaces and tabs; a file that can’t be read is listed as such, without Restore); Restore… asks first, then puts the sidebar back: tabs in both keep their page, tabs only in the backup come back asleep (with the page and history the Archive kept for them, leaving the Archive), tabs opened since go to the Archive, Spaces added since go; private windows and their tabs are left alone; Favorites over 12 go to Today (D33). When the sidebar can’t be backed up first, nothing changes. A running import stops first: none of the tabs it hasn’t shown yet go to the Archive, and its window says why (D65). Each restore is one step on the sidebar’s undo stack (TB-18): the toast’s Undo or ⌘Z, also in the Library, puts back the sidebar it replaced, a second ⌘Z goes further back and ⌘⇧Z restores again; earlier steps stay. After an undo, a window moved off a removed Space stays where it went, and the pages come back asleep"}, "FD-01": {"built": "yes", "note": "folders only in the pinned area (tree.canMoveTo)"}, "FD-02": {"built": "yes", "note": "New Folder at end of pinned in rename mode; New Folder with selected tabs"}, "FD-03": {"built": "yes", "note": "menu or double-click; Enter/blur saves, Escape cancels, empty keeps name"}, @@ -272,7 +272,7 @@ "DL-14": {"built": "yes", "note": "Windows/Linux: Ctrl+J opens the downloads flyout at the top left, beside the sidebar (Orbit has no top toolbar): the 12 newest with live progress, Keep/Discard, Resume/Retry/Cancel and Show in Folder; a click opens a file, rows drag out (DL-06), Show All opens the Library's Downloads section; Esc, Ctrl+J again or a click outside closes it. macOS: ⌘⇧J and ⌥⌘L open the Library's Downloads section (KS-12). The command bar has a \"Downloads\" action"}, "ST-01": {"built": "yes", "note": "own window (⌘, / Ctrl+,): opens or focuses one; remembers section and size; takes the opener Space's colours; sections come from a registry (src/renderer/src/settings/registry.ts); search filters rows"}, "ST-02": {"built": "yes", "note": "sections and common settings are command-bar actions (settings.*) that open Settings at the row; the keys lane adds Shortcuts"}, - "ST-03": {"built": "partial", "note": "default browser row, app-menu item and command-bar action (only an installed build can register; e2e uses a stub); Mini Window archive timing; Import button once the import lane registers import.open; Updates group: automatic checks switch, Check Now and the update feed address. Not built: UP-04's install-while-idle, hover-preview toggles (OT-26, P2), Open PDFs (P2)"}, + "ST-03": {"built": "partial", "note": "default browser row, app-menu item and command-bar action (only an installed build can register; e2e uses a stub); Mini Window archive timing; Import… opens the import flow (IM-01); Updates group: automatic checks switch, Check Now and the update feed address. Not built: UP-04's install-while-idle, hover-preview toggles (OT-26, P2), Open PDFs (P2)"}, "ST-04": {"built": "yes", "note": "list with Default first, rename any profile, New Profile…, delete with confirmation (off while a Space uses it, and says why)"}, "ST-05": {"built": "yes", "note": "12 hours (default), 24 hours, 7 days, 30 days per profile; no Never; other profiles follow Default until set"}, "ST-06": {"built": "yes", "note": "per-profile default engine (8 built-ins), custom engines with keyword and %s URL, Make default from the row menu; no extension override in Electron, so that row does not exist"}, @@ -283,14 +283,28 @@ "ST-10": {"built": "partial", "note": "tab sleep timer (TB-16), picture in picture, graphics acceleration; no site customizations, full-URL toggle, sounds or haptics, or edge-drag settings because those features don't exist yet"}, "ST-12": {"built": "partial", "note": "privacy promise (Orbit sends nothing); Clear browsing data per profile (⌘⇧⌫ / Ctrl+Shift+Delete): history by time range; cookies and site data, and the cache, always from all time (Electron has no range), so they start ticked only for All time; Secure DNS (Automatic, Off, Cloudflare, Google, Quad9); graphics acceleration lives in Advanced. No HTTPS-first or Safe Browsing (not in Electron), no per-site clearing here"}, "OB-01": {"built": "yes", "note": "a window of its own before any browser window on a first run (gated by settings.onboarded; e2e runs skip it unless ORBIT_ONBOARDING=1): Orbit's logo with the moon in motion (still with reduced motion), a welcome line, Get started; steps slide in (about 400 ms, none with reduced motion); Enter continues, Esc goes back; each step's title takes focus; progress dots are labelled. No sounds or music (optional), no haptics (Electron has no API), no first-window shimmer"}, - "OB-03": {"built": "partial", "note": "placeholder step that says import isn't available yet, because the import lane has no import.open; when it has, the step offers Import… (opens that flow in a browser window) and Do this later with the same weight. Not inline, no background import or counts, no link for people moving from another browser yet"}, + "OB-03": {"built": "partial", "note": "the import flow inline (IM-01 to IM-06): the browsers on this computer, several profiles allowed, the data types and the bookmark tree, a bookmarks file dropped or chosen, the paste route and the help for people moving from another browser, with Do this later weighing the same as Next. An import chosen there is read at once and added when Start browsing ends onboarding, where the Done screen sums it up; Skip setup adds nothing (D44). Letter tiles, not browser logos (as IM-02)"}, "OB-04": {"built": "yes", "note": "the Space theme picker (same ThemeEditor) for the first Space, applied live, with a preview of sidebar and frame, grain and intensity; Light / Dark / Match system (Match system preselected, Continue works at once); says it can be changed later. No haptics"}, "OB-05": {"built": "yes", "note": "16 web apps as letter tiles (no logos fetched), nothing preselected, at most 12; they become the first Space's profile's Favorites, unloaded until opened. No ordering by imported history (no import yet)"}, "OB-06": {"built": "not started", "note": "there is no ad and tracker blocker (OT-15), so the step is left out of the flow"}, "OB-07": {"built": "yes", "note": "explains what the default browser does; Make Orbit my default and Not now with the same weight; on Windows says what to pick in Default apps; status asked again when the window comes back to the front; no reminders later. Test runs use a stub"}, "OB-08": {"built": "yes", "note": "a Search with row on the Done screen: the Default profile's engine (ST-06's list), Google until changed"}, - "OB-09": {"built": "partial", "note": "Done screen sums up colours, Favorites, default browser and import; Start browsing (Back to browsing after a reset) opens the first window centred at about 86% of the screen with the colours and Favorites picked, command bar closed. No Orbit basics folder or local welcome tab (no local guide pages yet), no import counts"}, - "OB-14": {"built": "yes", "note": "Skip setup on every step but the last gives the spec's default first window: the theme step's colours and mode are undone and no Favorites are added (D44). The step and the picks so far are saved, so a quit part-way resumes there with them; Reset Onboarding (command bar only) runs the flow again over the open windows and deletes nothing"}, + "OB-09": {"built": "partial", "note": "Done screen sums up colours, Favorites, default browser and import (what will be added from where, e.g. \"6 bookmarks from bookmarks.html\", or that nothing was and where import lives); Start browsing (Back to browsing after a reset) opens the first window centred at about 86% of the screen with the colours and Favorites picked, command bar closed. No Orbit basics folder or local welcome tab (no local guide pages yet)"}, + "OB-14": {"built": "yes", "note": "Skip setup on every step but the last gives the spec's default first window: the theme step's colours and mode are undone and no Favorites are added (D44). The step, the theme and the Favorites picked so far are saved, so a quit part-way resumes there with them. An import chosen at the import step isn’t: it waits in memory for Start browsing, so a quit before then loses it, and the Done step says nothing was imported (it’s in Settings, under General); Reset Onboarding (command bar only) runs the flow again over the open windows and deletes nothing"}, + "IM-01": {"built": "partial", "note": "the import window (a child of the browser window, in its Space's colours): Back / Next / Cancel, Cancel at any step imports nothing, Enter is Next and Escape is Back, ⌘W closes it. Opened from onboarding (the same flow inline, OB-03), the command bar's Import from Another Browser, Settings → General → Import…, and on macOS the app menu's Import from Another Browser… (macOS-unverified). Not built: a ☰ entry on Windows/Linux (Orbit has no ☰ menu). In the progress view, Cancel is offered only while the source is read; once adding starts it offers Hide, and the bookmarks added are one undo step. The window isn’t modal: the browser window stays usable while it’s open, as IM-14 asks"}, + "IM-02": {"built": "partial", "note": "the browsers found on this computer (Chrome, Chromium, Brave, Edge, Vivaldi, Opera, Opera GX, Firefox; Safari on macOS), each a tile in its colour with its profile or profile count; those not installed aren't shown. Several profiles: a step to choose one or more, by name. Profiles come from Chromium's Local State (or a single Default folder, or Opera's own folder) and Firefox's profiles.ini; every file is read from a copy. Safari without Full Disk Access is shown as blocked, with the way round it (export a bookmarks file from Safari and import that). Tests search only made-up fixture folders. Letter tiles, not logos: Orbit doesn't ship other browsers' marks. Safari's reader (Bookmarks.plist, a binary property list) is tested on fixtures, macOS-unverified"}, + "IM-03": {"built": "partial", "note": "after the source and its profiles, what they hold as checkboxes, all on, with counts (\"6 bookmarks, as pinned tabs\", \"6 pages visited\"); types a source can't provide aren't shown, and a profile whose file the browser holds offers the rest; never passwords or cards (D52). Cancel (Do this later in onboarding) weighs the same as Next. Extensions are listed, off, as not importable yet (IM-09). Cookies (IM-10) aren’t offered. A file that is there but can’t be read is named with why, e.g. “Work: history couldn’t be read (the file is damaged)”, and the profile’s other types are still offered"}, + "IM-06": {"built": "yes", "note": "the last step: the profile (it gets the history) and the Space (its pinned section gets the bookmarks): the current Space by default, the first one in onboarding, or New Space… with a name. Several source profiles: \"Make a matching Orbit profile and Space for each\" makes them, in order, named after the source profiles, each getting its own profile's history. Private windows' profiles aren't offered and never get history"}, + "IM-04": {"built": "yes", "note": "the source's tree (Chromium's bookmarks bar, other and mobile bookmarks; Firefox's toolbar, menu, other and mobile bookmarks; Safari's favorites bar, menu and reading list; a file's own groups), several profiles each under its name, with a checkbox per folder and bookmark; everything chosen at first; Select all / none; \"4 of 6 bookmarks selected\". Addresses that can't be pinned show unticked with the reason. Folders below the top level start closed, and so does a top-level one holding over 1,000 bookmarks. The list isn’t virtualized: every row at the top level or in an open folder is drawn, so a large file shows quickly only when its bookmarks sit in such closed folders (20,000 in folders of 2,000 show in about 0.4 s). 20,000 bookmarks at the top level take about 2.9 s to show; 200,000 there, or 200 top-level folders of 1,000, about 30 s; 10,000 top-level folders about 6 s; and opening a large folder draws all its rows. Only the import window waits"}, + "IM-05": {"built": "partial", "note": "one pinned folder \"Imported from \" (\"Imported bookmarks\" from a file) at the end of the target Space's pinned section: the source's top-level groups and nested folders kept, in order, as closed sub-folders (the folder itself closed above 1,000 items); several profiles into one Space each get a sub-folder by name; each bookmark a pinned tab with its title and address, asleep, with no favicon or request until opened; bookmarklets, saved searches, browser-only and non-web addresses skipped and counted; Firefox tags and separators left out; no Favorites made; one undo step takes it all back. 2,000 bookmarks take about 0.3–0.4 s, made out of sight in parts and shown at once (IM-14), with the longest main-process stall about 40–50 ms. Not built: fetching a favicon when an imported row scrolls into view; its icon comes when the tab is first opened"}, + "IM-13": {"built": "yes", "note": "bookmarks HTML (the Netscape format; Chromium- and Firefox-style exports tested) and a .txt list of links, from From a file… or dropped on the import window or the sidebar. Bookmarks map as IM-05 into \"Imported bookmarks\"; links as IM-12 (a list over about 1 MB or 10,000 lines shows only its count and first lines); a file with neither says \"No bookmarks found in this file\" and imports nothing; a cut-off file imports what it holds. Files are read in a worker, in time linear in their size (D58): main’s longest stall is about 0.07 s for a 16 MB file or list, and about 0.2 s for a 60 MB file of 200,000 bookmarks. A file nested deeper than 64 folders keeps its deeper bookmarks in the 64th; a file gives at most 200,000 bookmarks (folders don’t count) and 20,000 folders, the bookmarks of folders past that kept in the folder that holds them; over 64 MB, every route, a paste included, says “This is too large to import (the limit is 64 MB)” without reading it; a read that fails says “Orbit couldn’t read this file”. No passwords CSV (D52)"}, + "IM-12": {"built": "yes", "note": "Paste a list of links (one address per line, \"Title — address\" with a dash, bar or tab, Markdown links, bullets): one pinned folder named by the user, a pinned tab per link in order, asleep; lines that aren't links are counted in the summary. ⌘V / Ctrl+V on the sidebar (outside a text field) with two or more links on the clipboard opens the same import with them filled in. A list over about 1 MB or 10,000 lines, pasted or from a file, isn’t put in the text field: the import window shows its count and first eight lines, read-only, with Clear, and imports it as it is. A paste over 64 MB as UTF-8 (22.5 million CJK characters, say) says “This is too large to import (the limit is 64 MB)”, as a file does, and none of it leaves the window it was pasted in. Clear also frees main’s copy of a held list. The other routes (a bookmarks file, notes, canvases, site styles) are explained in the import window's help, which reads nothing from another browser; passwords are out of scope (D52)"}, + "IM-14": {"built": "yes", "note": "progress per data type (a spinner, then a check), then a summary with counts per type and the skipped items with reasons; Done shows the imported folder in the sidebar (its Space, the pinned section open, the folder in view). Bookmarks are made out of sight, in parts, and their folder is shown whole, at once (D62): while 20,000 are added, the browser window’s longest freeze is under 0.1 s; 100,000 take about 34 s, with the window’s longest freeze about 0.34 s and main’s longest stall about 0.57 s (saving the state). A file the source browser holds (it can't be copied) shows \"Chrome is using some of its files. Quit Chrome and try again, or import the rest\" with Try again; the types it could read can still be imported. History shows the pages read, then added; Done says they're in History and the command bar; what History can't keep is listed with the reason. The summary lists every item left out with its reason: skipped bookmarks (also in a folder that holds nothing else), those past 200,000 bookmarks in one file, list or profile (folders don’t count, so the summary and the selection count agree), lines that aren’t links, and a profile’s file that couldn’t be read (“Work: history couldn’t be read (the file is damaged)”). Until the folder shows, nothing lists its tabs (D65): not the command bar (by title, by address or among recent tabs), the recent-tab switcher or the Library. One the user reached anyway and placed stays where they put it, and the folder leaves it out. A quit or a failure part-way takes back exactly the tabs the import made, by id, wherever they are, and nothing of the user’s; a crash part-way leaves none either, as the next start removes tabs that no section shows. Restore Data part-way stops the import first: its tabs are removed without going to the Archive, and the import window says “The import stopped: data was restored from a backup.” Copies of the other browser’s files go in a folder under Orbit’s user data, removed at quit (a copy still being made too) and cleared at startup (D58)"}, + "IM-07": {"built": "yes", "note": "Chromium's History, Firefox's places.sqlite and Safari's History.db (macOS-unverified), read from copies a window of rows at a time, into the chosen profile's history: each page with its title, visit and typed counts and its newest visits (up to 20). History (⌘Y / Ctrl+H) lists them and the command bar suggests and completes them at once; no tab is made and nothing loads. Left out: browsers' own pages, hidden pages, frame loads (Chromium subframes; Firefox embeds, framed links, downloads). History keeps the 50,000 most recent pages: a larger history keeps its newest and the summary counts the rest. 100,000 pages take about 2 s, with main's longest stall about 50–110 ms and none in the windows; the profile's history is written once, as a snapshot"}, + "IM-15": {"built": "yes", "note": "the same browser or file again makes a new \"Imported from (2)\" folder (then (3)…), never merging into or changing the first. History de-duplicates silently: a page already imported gains only the visits it lacked, so its counts aren't doubled. Passwords aren't imported (D52)"}, + "IM-16": {"built": "yes", "note": "after onboarding's import of a history, the Done step offers the six sites visited most (each by its most-visited page, or its home page when that's a search): the first three as Favorites, the rest as pinned tabs. One click takes one (another takes it back); Dismiss all drops them. They're added unloaded with the import when Start browsing ends onboarding, never with Skip setup (D44). Orbit can't tell web apps from other sites, so it ranks by visits; a suggested pinned tab is named after its site until first opened"}, + "IM-17": {"built": "yes", "note": "Export Pinned Items as Bookmarks… in the File menu (macOS-unverified: Orbit has no menu bar elsewhere), the command bar, and Settings → General → Import and export (with Include Today tabs) writes a standard bookmarks HTML file through the system's save dialog: the Favorites a folder (one per profile when there are several), each Space a folder of its pinned section with its folders nested as they are, a split's tabs in its place; Today's tabs only when chosen, as a Today folder in their Space. Orbit's own pages and private windows' profiles are left out. Copy All Links on a folder is the tabs lane's (it exists)"}, + "IM-09": {"built": "partial", "note": "A Chromium-family profile's extensions are listed on the types step, from its Preferences and Secure Preferences (read from copies; a name kept only in the manifest is read from it, in its default language): pinned ones first, the browser's own left out. The row is off and says they aren't importable yet; unpacked ones never come over. Not built: installing them fresh from the store with one summary confirmation, pinning the pinned ones, and the ad-blocker de-duplication (blockers are already marked), because the extensions lane's install path isn't on main yet. Firefox add-ons aren't listed (the spec covers Chromium-family sources)"}, "ST-11": {"built": "partial", "note": "Theme mode: Light / Dark / Match system (AP-01). Not built: window material (AP-10, Windows), app icon (AP-09), page defaults (P2)"}, "ST-17": {"built": "yes", "note": "Settings → About: logo, version (a click copies it), build date, Chromium and Electron versions, system, Check for updates, release notes, Copy build info. Licence and Open-source credits open credits.html, built with the app (D43): Orbit's and Electron's licences, a link to the LICENSES.chromium.html Electron ships, and the licence of each npm package whose code ships. Release notes and help need access to the private repository for now"}, "OB-13": {"built": "partial", "note": "command-bar Help, Report a Bug, What's New and Copy Build Info; macOS Help menu (Orbit Help, Report a Bug…, Release Notes, Troubleshooting → Copy Build Info). Help opens the project README; Report a Bug opens a new issue on the public tracker with the build info filled in. The shortcut sheet entry comes with OB-12 (keys lane)"}, diff --git a/scripts/features.mjs b/scripts/features.mjs index 99e68f92..6b1adc21 100644 --- a/scripts/features.mjs +++ b/scripts/features.mjs @@ -54,11 +54,16 @@ const ignore = (f) => f.startsWith('SOURCES') || f === 'FEATURES.md' const found = readdirSync(specDir).filter((f) => f.endsWith('.md') && !ignore(f)) const files = [...ORDER.filter((f) => found.includes(f)), ...found.filter((f) => !ORDER.includes(f)).sort()] -/** Arc-specific names stay in the spec; Orbit's checklist uses neutral names only. */ +/** + * The spec's own product names stay in the spec; Orbit's checklist uses neutral names only (D57). + * The name is spelled out, as in the clean-room test, so this file doesn't hold it. + */ +const NAME = ['A', 'r', 'c'].join('') const neutral = (title) => title - .replace(/\s*\((?:Arc[^)]*|[^)]*\bArc\b[^)]*)\)/g, '') - .replace(/\s*\[[^\]]*\bArc\b[^\]]*\]/g, '') + .replace(new RegExp(`\\s*\\((?:${NAME}[^)]*|[^)]*\\b${NAME}\\b[^)]*)\\)`, 'g'), '') + .replace(new RegExp(`\\s*\\[[^\\]]*\\b${NAME}\\b[^\\]]*\\]`, 'g'), '') + .replace(new RegExp(`\\bfrom ${NAME}\\b`, 'g'), 'from a browser Orbit can’t read') .trim() const sections = [] diff --git a/scripts/import-fixtures.ts b/scripts/import-fixtures.ts new file mode 100644 index 00000000..240ceb47 --- /dev/null +++ b/scripts/import-fixtures.ts @@ -0,0 +1,620 @@ +/** + * Builds the import lane's test fixtures (IM-*) from made-up data, into `tests/fixtures/import/`: + * - `files/bookmarks.htm`, `files/bookmarks-firefox.htm`: bookmarks files as browsers export them + * (folders, a bookmarklet, a browser's own page, entities); + * - `files/links.txt`: a list of links, plain and "Title — address"; + * - `files/malformed.htm`: an HTML file with no bookmarks in it; + * - `chrome/`, `firefox/`, `safari/`: the folders where those browsers keep their profiles, with only + * the files Orbit reads (IM-02 to IM-09): Chromium's `Local State`, `Bookmarks`, `History` and + * `Preferences`; Firefox's `profiles.ini` and `places.sqlite`; Safari's `Bookmarks.plist` (a binary + * property list) and `History.db`. + * HTML fixtures end in `.htm`, which the linter leaves alone: an export isn't meant to be valid HTML. + * + * Run it with `node scripts/import-fixtures.ts` after changing it, and commit what it writes. Tests + * also import its builders to make large files on the fly. Nothing here reads anything on this + * machine: every name, address and date is invented. + */ +import { mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { DatabaseSync } from 'node:sqlite' +import { fileURLToPath } from 'node:url' + +export const FIXTURE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', 'tests', 'fixtures', 'import') + +/** A made-up bookmark (with `url`) or folder (with `children`). */ +export interface FixtureNode { + title: string + url?: string + children?: FixtureNode[] +} + +/** A fixed date for every fixture (2024-03-01), so the files don't change between runs. */ +const ADDED = 1709251200 + +const escapeHtml = (text: string) => + text.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"') + +/** + * A Netscape bookmark file. `style` follows one exporter's habits: Chromium's (bookmarks bar flag, + * `

` after lists), or Firefox's (`

` descriptions, `
`, unquoted-looking extras). + */ +export function netscapeHtml(nodes: readonly FixtureNode[], style: 'chromium' | 'firefox' = 'chromium'): string { + const out = [ + '', + '', + '', + style === 'firefox' ? '' : '', + 'Bookmarks', + style === 'firefox' ? '

Bookmarks Menu

' : '

Bookmarks

', + '

', + ] + const walk = (list: readonly FixtureNode[], depth: number, top: boolean) => { + const pad = ' '.repeat(depth) + list.forEach((node, i) => { + if (node.children) { + const flag = top && i === 0 ? ' PERSONAL_TOOLBAR_FOLDER="true"' : '' + out.push(`${pad}

${escapeHtml(node.title)}

`) + out.push(`${pad}

`) + walk(node.children, depth + 1, false) + out.push(`${pad}

`) + if (style === 'firefox' && top) out.push(`${pad}

`) + } else if (node.url) { + const icon = style === 'chromium' ? ' ICON="data:image/png;base64,iVBORw0KGgo="' : '' + out.push(`${pad}
${escapeHtml(node.title)}`) + if (style === 'firefox') out.push(`${pad}
A note about ${escapeHtml(node.title)}`) + } + }) + } + walk(nodes, 1, true) + out.push('

', '') + return out.filter((line) => line !== '').join('\n') +} + +/** + * The bookmarks of IM-05's example, served from `base`: a bookmarks bar with "Work" (3 links) and + * "News" (2 links), other bookmarks with a nested folder, a bookmarklet and a browser's own page. + */ +export function sampleTree(base = 'https://example.com'): FixtureNode[] { + return [ + { + title: 'Bookmarks bar', + children: [ + { + title: 'Work', + children: [ + { title: 'Team calendar', url: `${base}/page/Calendar` }, + { title: 'Design docs', url: `${base}/page/Docs` }, + { title: 'Issue tracker', url: `${base}/page/Issues` }, + ], + }, + { + title: 'News', + children: [ + { title: 'Morning paper', url: `${base}/page/Paper` }, + { title: 'Tech & Science ', url: `${base}/page/Weekly` }, + ], + }, + ], + }, + { + title: 'Other bookmarks', + children: [ + { + title: 'Recipes', + children: [{ title: 'Bread', children: [{ title: 'Sourdough', url: `${base}/page/Sourdough` }] }], + }, + { title: 'Make it dark', url: 'javascript:document.body.style.background="#000"' }, + { title: 'Browser settings', url: 'chrome://settings/' }, + ], + }, + ] +} + +/** `count` bookmarks served from `base`: 10 folders of sub-folders of 20, plus a few loose ones. */ +export function manyBookmarks(count: number, base: string): FixtureNode[] { + const top: FixtureNode[] = [] + let n = 0 + const next = (): FixtureNode => { + n++ + return { title: `Page ${n}`, url: `${base}/page/p${n}` } + } + const loose = Math.min(5, count) + for (let f = 0; n < count - loose; f++) { + const folder: FixtureNode = { title: `Folder ${f + 1}`, children: [] } + for (let s = 0; s < 10 && n < count - loose; s++) { + const sub: FixtureNode = { title: `Folder ${f + 1}.${s + 1}`, children: [] } + for (let i = 0; i < 20 && n < count - loose; i++) sub.children?.push(next()) + folder.children?.push(sub) + } + top.push(folder) + } + while (n < count) top.push(next()) + return top +} + +/** A list of links as someone might copy them (IM-12): plain, "Title — address", and noise. */ +export const LINKS_TXT = [ + 'https://example.com/page/One', + 'Two — https://example.com/page/Two', + 'example.org/three', + '', + 'not a link at all', + '- [Four](https://example.com/page/Four)', + 'Five | https://example.com/page/Five', + '', +].join('\n') + +/** An HTML file that holds no bookmarks: a page, not an export (IM-13). */ +export const MALFORMED_HTML = ` +Not bookmarks +

Recipes

Flour, water, salt.

Bread
Bake it.
+` + +const DAY = 86_400_000 +/** The fixtures' "now" (2024-03-01), in ms: every visit is before it. */ +export const FIXTURE_NOW = ADDED * 1000 + +/** Browsers' clocks: Chromium counts microseconds from 1601, Firefox from 1970, Safari seconds from 2001. */ +export const chromeTime = (ms: number) => (BigInt(Math.round(ms)) + 11_644_473_600_000n) * 1000n +export const firefoxTime = (ms: number) => Math.round(ms) * 1000 +export const safariTime = (ms: number) => ms / 1000 - 978_307_200 + +/** A made-up page in a browser's history: its last visit (Unix ms), how often it was visited and typed. */ +export interface FixturePage { + url: string + title: string + visits: number + typed?: number + last: number + /** A page the browser keeps but doesn't list (a frame, a redirect). */ + hidden?: boolean +} + +/** A small history: sites visited often (IM-16's starters), a browser's own page, a hidden one. */ +export function sampleHistory(own = 'chrome://settings/'): FixturePage[] { + const at = (minutesAgo: number) => FIXTURE_NOW - minutesAgo * 60_000 + return [ + { url: 'https://mail.example.net/inbox', title: 'Inbox', visits: 300, typed: 5, last: at(1) }, + { url: 'https://news.example.org/', title: 'Example News', visits: 120, typed: 80, last: at(10) }, + { url: 'https://example.com/page/Calendar', title: 'Team calendar', visits: 42, typed: 30, last: at(60) }, + { url: 'https://docs.example.com/d/plan', title: 'Plan for the spring', visits: 17, typed: 2, last: at(2 * 1440) }, + { url: 'https://example.com/search?q=sourdough', title: 'sourdough - Search', visits: 1, last: at(5 * 1440) }, + { url: 'https://shop.example.com/cart', title: 'Cart', visits: 3, last: at(30 * 1440) }, + { url: own, title: 'Settings', visits: 4, last: at(1440) }, + { url: 'https://ads.example.com/frame', title: '', visits: 9, last: at(1440), hidden: true }, + ] +} + +/** `count` made-up pages a minute apart, most recent first. */ +export function manyPages(count: number, base = 'https://example.com'): FixturePage[] { + return Array.from({ length: count }, (_, i) => ({ + url: `${base}/h/${i + 1}`, + title: `History page ${i + 1}`, + visits: 1 + (i % 3), + last: FIXTURE_NOW - i * 60_000, + })) +} + +/** A made-up extension in a Chromium-family profile (IM-09). Location 1 is the store's, 4 unpacked, 5 the browser's own. */ +export interface FixtureExtension { + id: string + name: string + location?: number + pinned?: boolean +} + +export const SAMPLE_EXTENSIONS: FixtureExtension[] = [ + { id: 'a'.repeat(32), name: 'Example Blocker', pinned: true }, + { id: 'b'.repeat(32), name: 'Another Ad Blocker' }, + { id: 'c'.repeat(32), name: 'Notes Helper', pinned: true }, + { id: 'd'.repeat(32), name: 'My Unpacked Tool', location: 4 }, + { id: 'e'.repeat(32), name: 'Built-in Payments', location: 5 }, +] + +/** Writes a SQLite file afresh: small pages, one transaction, compacted. */ +function sqlite(path: string, fill: (db: DatabaseSync) => void): void { + mkdirSync(dirname(path), { recursive: true }) + rmSync(path, { force: true }) + const db = new DatabaseSync(path) + db.exec('PRAGMA page_size = 1024; PRAGMA journal_mode = DELETE') + db.exec('BEGIN') + fill(db) + db.exec('COMMIT') + db.exec('VACUUM') + db.close() +} + +const uuid = (n: number) => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` + +/** A Chromium-family profile's bookmark roots. */ +export interface ChromiumRoots { + bar: FixtureNode[] + other: FixtureNode[] + mobile: FixtureNode[] +} + +/** Chromium's `Bookmarks` file: JSON, indented by three as Chromium writes it. */ +export function chromiumBookmarksJson(roots: ChromiumRoots): string { + const added = String(chromeTime(FIXTURE_NOW - 30 * DAY)) + let id = 3 + const node = (n: FixtureNode): unknown => { + id++ + const common = { date_added: added, guid: uuid(id), id: String(id), name: n.title } + return n.children + ? { children: n.children.map(node), ...common, date_modified: added, type: 'folder' } + : { ...common, type: 'url', url: n.url } + } + const root = (n: number, name: string, children: FixtureNode[]) => ({ + children: children.map(node), + date_added: added, + date_modified: added, + guid: uuid(n), + id: String(n), + name, + type: 'folder', + }) + const bookmark_bar = root(1, 'Bookmarks bar', roots.bar) + const other = root(2, 'Other bookmarks', roots.other) + const synced = root(3, 'Mobile bookmarks', roots.mobile) + return JSON.stringify({ checksum: 'f'.repeat(32), roots: { bookmark_bar, other, synced }, version: 1 }, null, 3) +} + +/** Chromium's `History`: the `urls` and `visits` tables, with the columns Orbit reads. */ +export function writeChromiumHistory(path: string, pages: readonly FixturePage[]): void { + sqlite(path, (db) => { + db.exec(`CREATE TABLE meta(key LONGVARCHAR NOT NULL UNIQUE PRIMARY KEY, value LONGVARCHAR); + INSERT INTO meta VALUES ('version', '67'); + CREATE TABLE urls(id INTEGER PRIMARY KEY AUTOINCREMENT, url LONGVARCHAR, title LONGVARCHAR, + visit_count INTEGER DEFAULT 0 NOT NULL, typed_count INTEGER DEFAULT 0 NOT NULL, + last_visit_time INTEGER NOT NULL, hidden INTEGER DEFAULT 0 NOT NULL); + CREATE TABLE visits(id INTEGER PRIMARY KEY AUTOINCREMENT, url INTEGER NOT NULL, visit_time INTEGER NOT NULL, + from_visit INTEGER, transition INTEGER DEFAULT 0 NOT NULL, segment_id INTEGER, + visit_duration INTEGER DEFAULT 0 NOT NULL); + CREATE INDEX visits_url_index ON visits (url);`) + const url = db.prepare( + 'INSERT INTO urls (url, title, visit_count, typed_count, last_visit_time, hidden) VALUES (?, ?, ?, ?, ?, ?)', + ) + const visit = db.prepare('INSERT INTO visits (url, visit_time, transition) VALUES (?, ?, ?)') + for (const p of pages) { + const id = url.run(p.url, p.title, p.visits, p.typed ?? 0, chromeTime(p.last), p.hidden ? 1 : 0).lastInsertRowid + // Its latest visits (a fixture keeps three at most), a day apart; typed ones are transition 1. + for (let i = Math.min(p.visits, 3) - 1; i >= 0; i--) { + visit.run(id, chromeTime(p.last - i * DAY), i < (p.typed ?? 0) ? 1 : 0) + } + } + }) +} + +/** A Chromium-family profile folder (`folder` '.' puts its files in the browser's folder, as Opera does). */ +export interface ChromiumProfileFixture { + folder: string + name: string + bookmarks?: ChromiumRoots + history?: readonly FixturePage[] + extensions?: readonly FixtureExtension[] +} + +/** A Chromium-family browser's folder: `Local State` naming the profiles, then each profile's files. */ +export function writeChromium(dir: string, profiles: readonly ChromiumProfileFixture[], localState = true): void { + rmSync(dir, { recursive: true, force: true }) + if (localState) { + const cache = Object.fromEntries(profiles.map((p) => [p.folder, { is_using_default_name: false, name: p.name }])) + write( + join(dir, 'Local State'), + JSON.stringify({ profile: { info_cache: cache, profiles_order: profiles.map((p) => p.folder) } }), + ) + } + for (const p of profiles) { + const at = join(dir, p.folder) + if (p.bookmarks) write(join(at, 'Bookmarks'), chromiumBookmarksJson(p.bookmarks)) + if (p.history) writeChromiumHistory(join(at, 'History'), p.history) + if (p.extensions) { + const settings = Object.fromEntries( + p.extensions.map((x) => [ + x.id, + { + from_webstore: (x.location ?? 1) === 1, + location: x.location ?? 1, + manifest: { manifest_version: 3, name: x.name, version: '1.0.0' }, + state: 1, + }, + ]), + ) + const pinned = p.extensions.filter((x) => x.pinned).map((x) => x.id) + write(join(at, 'Preferences'), JSON.stringify({ extensions: { pinned_extensions: pinned, settings } })) + } + } +} + +/** A Firefox profile's bookmark roots, and its tags (tag → addresses), which aren't imported. */ +export interface FirefoxRoots { + toolbar: FixtureNode[] + menu: FixtureNode[] + unfiled: FixtureNode[] + mobile: FixtureNode[] + tags?: Record +} + +const revHost = (url: string) => { + try { + return `${[...new URL(url).hostname].reverse().join('')}.` + } catch { + return '' + } +} + +/** Firefox's `places.sqlite`: pages, visits and bookmarks, with the columns Orbit reads. */ +export function writePlaces(path: string, roots: FirefoxRoots, pages: readonly FixturePage[]): void { + sqlite(path, (db) => { + db.exec(`CREATE TABLE moz_places (id INTEGER PRIMARY KEY, url LONGVARCHAR, title LONGVARCHAR, rev_host LONGVARCHAR, + visit_count INTEGER DEFAULT 0, hidden INTEGER DEFAULT 0 NOT NULL, typed INTEGER DEFAULT 0 NOT NULL, + frecency INTEGER DEFAULT -1 NOT NULL, last_visit_date INTEGER, guid TEXT); + CREATE TABLE moz_bookmarks (id INTEGER PRIMARY KEY, type INTEGER, fk INTEGER DEFAULT NULL, parent INTEGER, + position INTEGER, title LONGVARCHAR, dateAdded INTEGER, lastModified INTEGER, guid TEXT); + CREATE TABLE moz_historyvisits (id INTEGER PRIMARY KEY, from_visit INTEGER, place_id INTEGER, + visit_date INTEGER, visit_type INTEGER, session INTEGER); + CREATE INDEX moz_historyvisits_placedateindex ON moz_historyvisits (place_id, visit_date);`) + const placeIds = new Map() + const addPlace = db.prepare( + 'INSERT INTO moz_places (url, title, rev_host, visit_count, hidden, typed, last_visit_date, guid) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + ) + const visit = db.prepare('INSERT INTO moz_historyvisits (place_id, visit_date, visit_type) VALUES (?, ?, ?)') + const place = (url: string, title: string, p?: FixturePage): number => { + const known = placeIds.get(url) + if (known !== undefined) return known + const last = p ? firefoxTime(p.last) : null + const row = addPlace.run( + url, + title, + revHost(url), + p?.visits ?? 0, + p?.hidden ? 1 : 0, + p?.typed ? 1 : 0, + last, + uuid(placeIds.size), + ) + const id = Number(row.lastInsertRowid) + placeIds.set(url, id) + // Its latest visits, a day apart; typed ones are visit type 2. + for (let i = Math.min(p?.visits ?? 0, 3) - 1; i >= 0; i--) { + visit.run(id, firefoxTime((p?.last ?? 0) - i * DAY), i < (p?.typed ?? 0) ? 2 : 1) + } + return id + } + for (const p of pages) place(p.url, p.title, p) + const mark = db.prepare( + 'INSERT INTO moz_bookmarks (id, type, fk, parent, position, title, dateAdded, lastModified, guid) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', + ) + const added = firefoxTime(FIXTURE_NOW - 30 * DAY) + let id = 0 + const add = (type: number, fk: number | null, parent: number, position: number, title: string, guid?: string) => { + id++ + mark.run(id, type, fk, parent, position, title, added, added, guid ?? `bookmark${String(id).padStart(4, '0')}`) + return id + } + const root = add(2, null, 0, 0, '', 'root________') + const menu = add(2, null, root, 0, 'menu', 'menu________') + const toolbar = add(2, null, root, 1, 'toolbar', 'toolbar_____') + const tags = add(2, null, root, 2, 'tags', 'tags________') + const unfiled = add(2, null, root, 3, 'unfiled', 'unfiled_____') + const mobile = add(2, null, root, 4, 'mobile', 'mobile______') + const walk = (list: readonly FixtureNode[], parent: number) => + list.forEach((n, i) => { + if (n.children) walk(n.children, add(2, null, parent, i, n.title)) + else if (n.url) add(1, place(n.url, n.title), parent, i, n.title) + }) + walk(roots.toolbar, toolbar) + walk(roots.menu, menu) + // A separator ends the menu, as in a new profile. + add(3, null, menu, roots.menu.length, '') + walk(roots.unfiled, unfiled) + walk(roots.mobile, mobile) + Object.entries(roots.tags ?? {}).forEach(([tag, urls], i) => { + const folder = add(2, null, tags, i, tag) + urls.forEach((u, j) => { + add(1, place(u, ''), folder, j, '') + }) + }) + }) +} + +/** Firefox's folder: `profiles.ini` (the install's default first; `stale` names a folder that is gone), then each profile. */ +export function writeFirefox( + dir: string, + profiles: readonly { path: string; name: string; roots: FirefoxRoots; history: readonly FixturePage[] }[], + stale?: string, +): void { + rmSync(dir, { recursive: true, force: true }) + const ini = [`[Install4F96D1932A9F858E]`, `Default=${profiles[0]?.path ?? ''}`, 'Locked=1', ''] + const listed = [...(stale ? [{ path: stale, name: 'old' }] : []), ...profiles] + listed.forEach((p, i) => { + ini.push(`[Profile${i}]`, `Name=${p.name}`, 'IsRelative=1', `Path=${p.path}`, '') + }) + ini.push('[General]', 'StartWithLastProfile=1', 'Version=2', '') + write(join(dir, 'profiles.ini'), ini.join('\n')) + for (const p of profiles) writePlaces(join(dir, p.path, 'places.sqlite'), p.roots, p.history) +} + +/** + * A binary property list ("bplist00"), the format of Safari's `Bookmarks.plist`: dictionaries, + * arrays, strings, integers and booleans; two-byte references, four-byte offsets. + */ +export function bplist(root: unknown): Buffer { + const objects: Buffer[] = [] + const int = (n: number) => + n < 0x100 + ? Buffer.from([0x10, n]) + : n < 0x10000 + ? Buffer.from([0x11, n >> 8, n & 0xff]) + : Buffer.from([0x12, n >>> 24, (n >> 16) & 0xff, (n >> 8) & 0xff, n & 0xff]) + const head = (kind: number, n: number) => + n < 15 ? Buffer.from([(kind << 4) | n]) : Buffer.concat([Buffer.from([(kind << 4) | 0xf]), int(n)]) + const refs = (list: number[]) => { + const b = Buffer.alloc(list.length * 2) + list.forEach((r, i) => { + b.writeUInt16BE(r, i * 2) + }) + return b + } + const encode = (v: unknown): number => { + const i = objects.push(Buffer.alloc(0)) - 1 + if (Array.isArray(v)) { + const items = v.map(encode) + objects[i] = Buffer.concat([head(0xa, items.length), refs(items)]) + } else if (v && typeof v === 'object') { + const entries = Object.entries(v) + const keys = entries.map(([k]) => encode(k)) + const values = entries.map(([, x]) => encode(x)) + objects[i] = Buffer.concat([head(0xd, entries.length), refs(keys), refs(values)]) + } else if (typeof v === 'string') { + const ascii = Buffer.byteLength(v, 'utf8') === v.length + const body = ascii ? Buffer.from(v, 'latin1') : Buffer.from(v, 'utf16le').swap16() + objects[i] = Buffer.concat([head(ascii ? 0x5 : 0x6, v.length), body]) + } else if (typeof v === 'boolean') { + objects[i] = Buffer.from([v ? 0x09 : 0x08]) + } else { + const b = Buffer.alloc(9) + b[0] = 0x13 + b.writeBigInt64BE(BigInt(Math.round(Number(v))), 1) + objects[i] = b + } + return i + } + encode(root) + const offsets = Buffer.alloc(objects.length * 4) + let at = 8 + objects.forEach((o, i) => { + offsets.writeUInt32BE(at, i * 4) + at += o.length + }) + const trailer = Buffer.alloc(32) + trailer[6] = 4 + trailer[7] = 2 + trailer.writeBigUInt64BE(BigInt(objects.length), 8) + trailer.writeBigUInt64BE(BigInt(at), 24) + return Buffer.concat([Buffer.from('bplist00', 'latin1'), ...objects, offsets, trailer]) +} + +/** Safari's folder: `Bookmarks.plist` (favorites bar, bookmarks menu, reading list) and `History.db`. */ +export function writeSafari( + dir: string, + roots: { bar: FixtureNode[]; menu: FixtureNode[]; readingList: FixtureNode[] }, + pages: readonly FixturePage[], +): void { + rmSync(dir, { recursive: true, force: true }) + let n = 0 + const list = (title: string, children: readonly FixtureNode[]) => ({ + Children: children.map(node), + Title: title, + WebBookmarkType: 'WebBookmarkTypeList', + WebBookmarkUUID: uuid(++n), + }) + const node = (b: FixtureNode): unknown => + b.children + ? list(b.title, b.children) + : { + URIDictionary: { title: b.title }, + URLString: b.url, + WebBookmarkType: 'WebBookmarkTypeLeaf', + WebBookmarkUUID: uuid(++n), + } + const plist = { + Children: [ + { Title: 'History', WebBookmarkIdentifier: 'History', WebBookmarkType: 'WebBookmarkTypeProxy' }, + list('BookmarksBar', roots.bar), + list('BookmarksMenu', roots.menu), + { ...list('com.apple.ReadingList', roots.readingList), ShouldOmitFromUI: true }, + ], + Title: '', + WebBookmarkFileVersion: 1, + WebBookmarkType: 'WebBookmarkTypeList', + } + mkdirSync(dir, { recursive: true }) + writeFileSync(join(dir, 'Bookmarks.plist'), bplist(plist)) + sqlite(join(dir, 'History.db'), (db) => { + db.exec(`CREATE TABLE history_items (id INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT NOT NULL UNIQUE, + domain_expansion TEXT NULL, visit_count INTEGER NOT NULL); + CREATE TABLE history_visits (id INTEGER PRIMARY KEY AUTOINCREMENT, history_item INTEGER NOT NULL, + visit_time REAL NOT NULL, title TEXT NULL, load_successful BOOLEAN NOT NULL DEFAULT 1);`) + const item = db.prepare('INSERT INTO history_items (url, visit_count) VALUES (?, ?)') + const visit = db.prepare('INSERT INTO history_visits (history_item, visit_time, title) VALUES (?, ?, ?)') + for (const p of pages) { + if (p.hidden) continue + const id = item.run(p.url, p.visits).lastInsertRowid + for (let i = Math.min(p.visits, 3) - 1; i >= 0; i--) visit.run(id, safariTime(p.last - i * DAY), p.title || null) + } + }) +} + +/** The made-up browsers of the made-up computer the tests search (IM-02). */ +export function buildBrowsers(root = FIXTURE_ROOT): void { + const [bar, other] = sampleTree() + const weather = { title: 'Weather', url: 'https://weather.example.org/' } + writeChromium(join(root, 'chrome'), [ + { + folder: 'Default', + name: 'Work', + bookmarks: { bar: bar?.children ?? [], other: other?.children ?? [], mobile: [] }, + history: sampleHistory(), + extensions: SAMPLE_EXTENSIONS, + }, + { + folder: 'Profile 1', + name: 'Personal', + bookmarks: { + bar: [weather], + other: [{ title: 'Travel', children: [{ title: 'Trains', url: 'https://example.com/page/Trains' }] }], + mobile: [{ title: 'Recipes on the go', url: 'https://example.com/page/Mobile' }], + }, + history: [{ url: weather.url, title: 'Weather', visits: 12, typed: 12, last: FIXTURE_NOW - 3 * 60_000 }], + }, + ]) + writeFirefox( + join(root, 'firefox'), + [ + { + path: 'abcd1234.default-release', + name: 'default-release', + roots: { + toolbar: [{ title: 'Most Visited', url: 'place:sort=8&maxResults=10' }, ...(bar?.children ?? [])], + menu: other?.children ?? [], + unfiled: [{ title: 'Read later', url: 'https://example.com/page/Later' }], + mobile: [], + tags: { bread: ['https://example.com/page/Sourdough'] }, + }, + history: sampleHistory('about:preferences'), + }, + ], + 'zzzz0000.old', + ) + writeSafari( + join(root, 'safari'), + { + bar: bar?.children ?? [], + menu: [{ title: 'Recipes', children: [{ title: 'Sourdough', url: 'https://example.com/page/Sourdough' }] }], + readingList: [{ title: 'A long read', url: 'https://example.com/page/Long' }], + }, + sampleHistory('favorites://'), + ) +} + +function write(path: string, text: string): void { + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, text) +} + +/** Writes every fixture under `root`. */ +export function buildFixtures(root = FIXTURE_ROOT): void { + write(join(root, 'files', 'bookmarks.htm'), netscapeHtml(sampleTree())) + write(join(root, 'files', 'bookmarks-firefox.htm'), netscapeHtml(sampleTree(), 'firefox')) + write(join(root, 'files', 'links.txt'), LINKS_TXT) + write(join(root, 'files', 'malformed.htm'), MALFORMED_HTML) + buildBrowsers(root) +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + buildFixtures() + console.log(`import fixtures written to ${FIXTURE_ROOT}`) +} diff --git a/src/main/archive/restore-data.ts b/src/main/archive/restore-data.ts index 1062408e..4213e823 100644 --- a/src/main/archive/restore-data.ts +++ b/src/main/archive/restore-data.ts @@ -1,5 +1,6 @@ import type { BackupRestoreResult, BackupUndoResult } from '../../shared/commands/archive' import type { MainContext } from '../context' +import { stopImportsForRestore } from '../import/unplaced' import type { BackupStore } from './backups' import { sidebarUndo } from './inbox' import { buildSnapshot, restoreSidebar, type SidebarSnapshot } from './snapshot' @@ -51,6 +52,8 @@ export class RestoreData { console.error('[backups] could not back up the sidebar before a restore; nothing was changed', err) return { restored: false, reason: 'backup-failed' } } + // A running import stops first (D65 (3)), so what this restore replaces holds nothing of it. + stopImportsForRestore() const at = this.now() const state = this.ctx.store.state const replaced = buildSnapshot(state, at) diff --git a/src/main/archive/snapshot.ts b/src/main/archive/snapshot.ts index 50ff4d2b..74593adb 100644 --- a/src/main/archive/snapshot.ts +++ b/src/main/archive/snapshot.ts @@ -19,6 +19,7 @@ import type { import { tabTitle } from '../../shared/tabs' import { sectionOf, sidebarTabOrder, splitLeaves } from '../../shared/tree' import type { MainContext } from '../context' +import { stopImportsForRestore } from '../import/unplaced' import { normalizeState } from '../persist' import { dedupeDisplayed, mostRecentTab, shownElsewhere } from '../tabs/model' import type { SavedHistory } from '../tabs/runtime' @@ -614,7 +615,8 @@ function historyOf(ctx: Pick, tabId: TabId): SavedHistory | * Puts the sidebar back as a backup has it (AR-11): one state update (`planRestore`, * `applyRestore`), then what follows outside the state. Returning tabs get their history before * they can wake; tabs that left hand theirs to the Archive and close their page; tabs now in - * another profile start again in its session. + * another profile start again in its session. A running import stops first: the tabs it made out + * of sight go, and never reach the Archive (D65 (3)). */ export function restoreSidebar( ctx: Pick, @@ -622,6 +624,7 @@ export function restoreSidebar( snap: SidebarSnapshot, now = Date.now(), ): RestorePlan { + stopImportsForRestore() const state = ctx.store.state const hot = new Map() for (const a of state.archive) if (a.tabId) hot.set(a.tabId, [...(hot.get(a.tabId) ?? []), a]) diff --git a/src/main/commandbar/sources.ts b/src/main/commandbar/sources.ts index cd39c489..0d165f8b 100644 --- a/src/main/commandbar/sources.ts +++ b/src/main/commandbar/sources.ts @@ -9,6 +9,7 @@ import type { ArchivedTab, FolderId, ItemId, OrbitState, ProfileId, SpaceId, Tab import { splitLeaves } from '../../shared/tree' import { type ArchiveSource, archiveSource } from '../archive/inbox' import { hostOf, urlKeyOf } from '../history/store' +import { isUnplacedImportTab, unplacedImportVersion } from '../import/unplaced' import { initials } from './match' import type { ActionDoc, ArchiveDoc, FolderDoc, SpaceDoc, TabDoc } from './rank' @@ -60,14 +61,18 @@ export class SourceCache { private names: Record = {} private actionDocs: ActionDoc[] | undefined - /** Every tab in every Space (pinned, Favorites, Today, folders, splits). */ + /** + * Every tab in every Space (pinned, Favorites, Today, folders, splits), and a Peek's or a mini window's. Not an import's + * tabs that no section shows yet: nothing may reach them until their folder is shown (D65 (1)). + */ tabs(state: OrbitState): TabDoc[] { const layout = this.layoutOf(state) - const refs = [state.tabs, layout] + const refs = [state.tabs, layout, unplacedImportVersion()] if (sameRefs(refs, this.tabsRef)) return this.tabDocs this.tabsRef = refs const docs: TabDoc[] = [] for (const tab of Object.values(state.tabs)) { + if (isUnplacedImportTab(tab.id) && !layout.section.has(tab.id)) continue let doc = this.tabDocCache.get(tab) if (!doc) { doc = makeTabDoc(tab, layout.section.get(tab.id)) diff --git a/src/main/features.ts b/src/main/features.ts index 0f78cd10..71edf733 100644 --- a/src/main/features.ts +++ b/src/main/features.ts @@ -38,7 +38,7 @@ export function registerFeatures(ctx: MainContext): void { registerExtensions(ctx) registerDownloads(ctx) registerSettings(ctx) - registerImport(ctx) + registerImport(ctx, history) registerUpdates(ctx) // Last, so its guard on new tabs and windows goes before every other (Peek's, Mini Window's). registerInternalPages(ctx) diff --git a/src/main/history/store.ts b/src/main/history/store.ts index ae486c4d..fe253527 100644 --- a/src/main/history/store.ts +++ b/src/main/history/store.ts @@ -66,6 +66,16 @@ export interface HostInfo { lastVisit: number } +/** A page from another browser's history (IM-07, added by the import lane). */ +export interface ImportedPage { + url: string + title: string + visitCount: number + typedCount: number + /** Its latest visit times, in ms (the newest MAX_VISITS_PER_ENTRY are kept). */ + visits: number[] +} + export interface HistoryStoreOptions { now?: () => number /** Background compaction once appended bytes pass max(this, last snapshot size). Default 1 MiB. */ @@ -249,6 +259,64 @@ export class HistoryStore { this.write(rec) } + /** + * Adds pages imported from another browser (IM-07, import lane), after `load()`: one pass, no log + * record per visit. A visit already here isn't added again, so importing the same history twice + * changes nothing (IM-15). Call it a chunk at a time, then `finishImport()`. Returns the pages + * that were new or gained visits. + */ + importPages(pages: readonly ImportedPage[]): number { + if (!this.isLoaded) throw new Error('History isn’t loaded yet') + if (this.dropped) return 0 + let changed = 0 + for (const p of pages) { + if (!isRecordable(p.url)) continue + const times = [...new Set(p.visits.filter(Number.isFinite))].sort((a, b) => a - b) + const recent = times.slice(-MAX_VISITS_PER_ENTRY) + const known = this.map.get(p.url) + // Visits this page already has mean it was imported before: only its newer visits are added + // (an entry keeps its newest visits only, so older ones may be counted already). + const before = known && recent.some((at) => known.visits.includes(at)) ? known : undefined + const fresh = before + ? recent.filter( + (at) => + !before.visits.includes(at) && (before.visits.length < MAX_VISITS_PER_ENTRY || at > before.visits[0]!), + ) + : recent + const first = fresh[0] + const last = fresh[fresh.length - 1] + if (first === undefined || last === undefined) continue + const visits = before ? fresh.length : Math.max(Number(p.visitCount) || 0, times.length) + const typed = before ? 0 : Math.min(Math.max(0, Number(p.typedCount) || 0), visits) + const e = this.entry(this.map, p.url, first) + const title = p.title.trim().slice(0, MAX_TITLE) + if (!e.title && title) { + e.title = title + e.titleLower = title.toLowerCase() + } + e.visitCount += visits + e.typedCount += typed + e.firstVisit = Math.min(e.firstVisit, first) + e.lastVisit = Math.max(e.lastVisit, last) + for (const at of fresh) pushVisit(e.visits, at) + this.countHost(e, visits, typed, e.lastVisit) + // A log that can't be read gets each addition in its side file; otherwise finishImport's snapshot holds them. + if (this.phase !== 'ready') { + this.write({ t: 'e', u: p.url, ti: e.title, vc: visits, tc: typed, fv: first, v: fresh }) + } + changed++ + } + if (changed) this.changed() + return changed + } + + /** After `importPages`: keeps the newest MAX_ENTRIES entries and writes them as a snapshot. */ + async finishImport(): Promise { + this.trim() + await this.idle() + await this.compact() + } + /** Forgets everything and removes the files (the profile was deleted). */ async drop(): Promise { this.dropped = true diff --git a/src/main/import/bookmarks.ts b/src/main/import/bookmarks.ts new file mode 100644 index 00000000..52f38aff --- /dev/null +++ b/src/main/import/bookmarks.ts @@ -0,0 +1,258 @@ +/** + * Imported bookmarks become pinned items (IM-05): one new pinned folder at the end of the target + * Space's pinned section, holding a pinned tab per bookmark and a sub-folder per bookmark folder, + * in their order. Pure state helpers (draft in, ids out), unit-tested directly. + */ +import { freeze } from 'immer' +import type { BookmarkNode } from '../../shared/commands/import' +import { newId } from '../../shared/ids' +import type { Folder, FolderId, OrbitState, ProfileId, SpaceId, Tab, TabId } from '../../shared/state' +import { insertItem } from '../../shared/tree' +import { fallbackTitle } from '../../shared/url' +import { makeTab } from '../tabs/model' + +/** An import holding more items than this shows its folder collapsed (IM-05). */ +export const COLLAPSE_OVER = 1000 + +export interface PlacedFolder { + /** The new top folder ('' when nothing was added). */ + folderId: FolderId + tabIds: TabId[] + folderIds: FolderId[] + /** Bookmarks left out, by reason. */ + skipped: Map +} + +/** The name, or "name (2)", "name (3)"… when a top-level pinned folder of the Space has it (IM-15). */ +export function freeFolderName(state: OrbitState, spaceId: SpaceId, base: string): string { + const taken = new Set((state.spaces[spaceId]?.pinned ?? []).map((id) => state.folders[id]?.name)) + if (!taken.has(base)) return base + for (let n = 2; ; n++) if (!taken.has(`${base} (${n})`)) return `${base} (${n})` +} + +/** One step of placing an import, in order: a folder or a pinned tab, under its parent's key. */ +export type PlaceStep = + | { kind: 'folder'; key: number; parent: number; name: string } + | { kind: 'tab'; parent: number; url: string; title: string } + +/** Where an import's bookmarks go, worked out before anything is added. */ +export interface Placement { + /** The new top folder (key 0, parent -1) first, then folders and tabs in the file's order. */ + steps: PlaceStep[] + tabs: number + folders: number + /** Bookmarks left out, by reason. */ + skipped: Map +} + +/** + * Plans folder `name` holding `nodes` (IM-05). Bookmarks that can't be pinned tabs are only counted, + * and a folder with no tab under it (one that held only those) isn't added (F3). + */ +export function planPlacement(name: string, nodes: readonly BookmarkNode[]): Placement { + const plan: Placement = { + steps: [{ kind: 'folder', key: 0, parent: -1, name }], + tabs: 0, + folders: 0, + skipped: new Map(), + } + const empty = new Set() + let keys = 0 + /** Plans `list` into folder `parent`; returns how many tabs it holds. */ + const walk = (list: readonly BookmarkNode[], parent: number): number => { + let tabs = 0 + for (const node of list) { + if (node.children) { + const key = ++keys + const step: PlaceStep = { + kind: 'folder', + key, + parent, + name: node.title.trim().slice(0, 200) || 'Untitled folder', + } + plan.steps.push(step) + const inside = walk(node.children, key) + if (inside === 0) empty.add(step) + else plan.folders++ + tabs += inside + } else if (node.url) { + if (node.skip) { + plan.skipped.set(node.skip, (plan.skipped.get(node.skip) ?? 0) + 1) + continue + } + plan.steps.push({ + kind: 'tab', + parent, + url: node.url, + title: node.title.trim().slice(0, 400) || fallbackTitle(node.url), + }) + plan.tabs++ + tabs++ + } + } + return tabs + } + walk(nodes, 0) + // A folder with no tab under it has only such folders under it: they all go. + if (empty.size > 0) plan.steps = plan.steps.filter((step) => !empty.has(step)) + return plan +} + +/** + * One Space's folder of an import, made out of sight (D62 (3)). `build` makes the plan's steps a part at a time, and + * each part's tabs go into the state where no section shows them (`addUnplacedTabs`); `attachFolders` then adds the + * folders and shows the whole folder at once. Until then the sidebar shows none of it and no list of open tabs lists + * its tabs (D65 (1)), so nothing can be moved into its folders. A tab the user reached anyway and placed stays where + * they put it: `keepOnly` leaves it out of the folder at the attach (D65 (2)). A quit or a failure takes back exactly + * the tabs added, wherever they are. Tabs are asleep, have no favicon (it comes when the page first loads) and count + * as never used. + */ +export class DetachedFolder { + /** The top folder first, then its folders in the plan's order. */ + readonly folders: Folder[] = [] + readonly tabs: Tab[] = [] + private readonly byKey = new Map() + private made = 0 + + constructor( + private readonly plan: Placement, + readonly spaceId: SpaceId, + /** The profile its tabs are made in: the Space's when the import started. */ + readonly profileId: ProfileId, + private readonly now: number, + ) {} + + /** Every step is made. */ + get whole(): boolean { + return this.made >= this.plan.steps.length + } + + /** The tabs it holds when whole. */ + get total(): number { + return this.plan.tabs + } + + /** Makes steps until `tabs` more tabs are made, or the plan ends; returns those tabs. */ + build(tabs: number): Tab[] { + const { plan, now } = this + const out: Tab[] = [] + for (; this.made < plan.steps.length && out.length < tabs; this.made++) { + const step = plan.steps[this.made] as PlaceStep + if (step.kind === 'folder') { + // Sub-folders start closed; the top folder too when it holds more than COLLAPSE_OVER items. + const top = step.parent === -1 + const folder: Folder = { + id: newId('f'), + name: step.name, + children: [], + collapsed: top ? plan.tabs + plan.folders > COLLAPSE_OVER : true, + createdAt: now, + } + this.folders.push(folder) + this.byKey.set(step.key, folder) + if (!top) this.byKey.get(step.parent)?.children.push(folder.id) + continue + } + const tab: Tab = makeTab( + { + spaceId: this.spaceId, + profileId: this.profileId, + url: step.url, + title: step.title, + pinnedUrl: step.url, + pinnedResolvedUrl: step.url, + pinnedTitle: step.title, + createdAt: now, + // Never used in Orbit: recent-tab lists and Ctrl+Tab don't fill up with them. + lastActiveAt: 0, + }, + now, + ) + out.push(tab) + this.byKey.get(step.parent)?.children.push(tab.id) + } + this.tabs.push(...out) + return out + } + + /** + * Keeps only the tabs `keep` accepts, leaving the others out of its folders: at the attach, those gone from the state + * and those the user placed somewhere meanwhile (D65 (2)). Returns how many are left. + */ + keepOnly(keep: (id: TabId) => boolean): number { + const gone = new Set(this.tabs.filter((t) => !keep(t.id)).map((t) => t.id)) + if (gone.size > 0) { + for (const f of this.folders) f.children = f.children.filter((id) => !gone.has(id)) + this.tabs.splice(0, this.tabs.length, ...this.tabs.filter((t) => !gone.has(t.id))) + } + return this.tabs.length + } +} + +/** Adds tabs an import made where no section shows them, frozen first (Immer then skips walking each one). Draft helper. */ +export function addUnplacedTabs(d: OrbitState, tabs: readonly Tab[]): void { + for (const t of tabs) d.tabs[t.id] = freeze(t) +} + +/** + * Shows whole folders at once (D62 (3)): each one's folders are added, its top folder named `names[i]` and put at the + * end of its Space's pinned section. Their tabs are in the state already (`addUnplacedTabs`). A Space deleted meanwhile + * gets nothing. Returns the folders shown. Draft helper. + */ +export function attachFolders( + d: OrbitState, + built: readonly DetachedFolder[], + names: readonly string[], +): { folderId: FolderId; spaceId: SpaceId; tabs: number }[] { + const added: { folderId: FolderId; spaceId: SpaceId; tabs: number }[] = [] + built.forEach((b, i) => { + const space = d.spaces[b.spaceId] + const root = b.folders[0] + if (!space || !root || !b.whole || b.tabs.length === 0) return + root.name = names[i] ?? root.name + for (const f of b.folders) d.folders[f.id] = freeze(f) + // The Space moved to another profile meanwhile (PR-03): its new tabs go with it. + if (space.profileId !== b.profileId) { + for (const t of b.tabs) { + const tab = d.tabs[t.id] + if (tab) tab.profileId = space.profileId + } + } + insertItem(d, { kind: 'pinned', spaceId: b.spaceId }, space.pinned.length, root.id) + added.push({ folderId: root.id, spaceId: b.spaceId, tabs: b.tabs.length }) + }) + return added +} + +/** + * Adds folder `name` with `nodes` to the end of the Space's pinned section in one go (IM-05). When + * every bookmark is left out, nothing is added, not even an empty folder. Draft helper. + */ +export function placeBookmarks( + d: OrbitState, + spaceId: SpaceId, + name: string, + nodes: readonly BookmarkNode[], + now: number, +): PlacedFolder | undefined { + const space = d.spaces[spaceId] + if (!space) return undefined + const plan = planPlacement(name, nodes) + const placed: PlacedFolder = { folderId: '', tabIds: [], folderIds: [], skipped: plan.skipped } + if (plan.tabs === 0) return placed + const folder = new DetachedFolder(plan, spaceId, space.profileId, now) + addUnplacedTabs(d, folder.build(plan.tabs)) + attachFolders(d, [folder], [name]) + placed.folderId = folder.folders[0]?.id ?? '' + placed.folderIds = folder.folders.map((f) => f.id) + placed.tabIds = folder.tabs.map((t) => t.id) + return placed +} + +/** Reasons for left-out bookmarks (or links), as summary lines. */ +export function skippedLines( + skipped: Map, + [one, many]: readonly [string, string] = ['bookmark', 'bookmarks'], +): { what: string; count: number; reason: string }[] { + return [...skipped].map(([reason, count]) => ({ what: count === 1 ? one : many, count, reason })) +} diff --git a/src/main/import/browsers.ts b/src/main/import/browsers.ts new file mode 100644 index 00000000..fb8f3711 --- /dev/null +++ b/src/main/import/browsers.ts @@ -0,0 +1,210 @@ +import { access, open, readFile } from 'node:fs/promises' +import { isAbsolute, join } from 'node:path' +import type { + BookmarkNode, + BrowserKind, + ImportDataType, + ImportSource, + ProfileContents, +} from '../../shared/commands/import' +import { chromiumBookmarks, chromiumProfiles } from './chromium' +import { LockedError, type Scratch, unreadableReason, withDatabase } from './copy' +import { chromiumExtensions } from './extensions' +import { firefoxProfiles, readFirefoxBookmarks } from './firefox' +import { countHistory, historySql } from './history' +import { countBookmarks } from './netscape' +import type { Place } from './places' +import { parseBplist, safariBookmarks } from './safari' + +/** Each browser's name: on its tile, and in "Imported from " (IM-05). */ +export const BROWSER_NAMES: Record = { + chrome: 'Chrome', + chromium: 'Chromium', + brave: 'Brave', + edge: 'Edge', + vivaldi: 'Vivaldi', + opera: 'Opera', + 'opera-gx': 'Opera GX', + firefox: 'Firefox', + safari: 'Safari', +} + +/** Safari's files need Full Disk Access on macOS; this is the way round it (IM-02). */ +export const SAFARI_BLOCKED = + 'Needs Full Disk Access. Or export bookmarks from Safari (File → Export → Bookmarks) and import that file.' + +/** A profile found on this computer. Its folder stays in the main process. */ +export interface FoundProfile { + id: string + name: string + dir: string +} + +export interface FoundSource extends Omit { + profiles: FoundProfile[] +} + +/** What the import window may know of a source: no folders. */ +export function publicSource({ profiles, ...source }: FoundSource): ImportSource { + return { ...source, profiles: profiles.map(({ id, name }) => ({ id, name })) } +} + +const exists = (path: string) => + access(path).then( + () => true, + () => false, + ) + +/** Whether a file opens for reading: false when it's missing, 'denied' without permission. */ +async function readable(path: string): Promise { + try { + await (await open(path, 'r')).close() + return true + } catch (err) { + const code = (err as NodeJS.ErrnoException).code + return code === 'EPERM' || code === 'EACCES' ? 'denied' : false + } +} + +const hasChromiumData = async (dir: string) => + (await exists(join(dir, 'Bookmarks'))) || (await exists(join(dir, 'History'))) + +/** The browser at a place and its profiles, or undefined when nothing there can be read. */ +async function findAt({ browser, dir }: Place): Promise | undefined> { + if (browser === 'firefox') { + const ini = await readFile(join(dir, 'profiles.ini'), 'utf8').catch(() => undefined) + if (ini === undefined) return undefined + const profiles: FoundProfile[] = [] + for (const p of firefoxProfiles(ini)) { + const at = p.relative || !isAbsolute(p.path) ? join(dir, p.path) : p.path + if (await exists(join(at, 'places.sqlite'))) profiles.push({ id: p.path, name: p.name, dir: at }) + } + return profiles.length > 0 ? { browser, profiles } : undefined + } + if (browser === 'safari') { + const marks = await readable(join(dir, 'Bookmarks.plist')) + const history = await readable(join(dir, 'History.db')) + if (marks === 'denied' || history === 'denied') return { browser, profiles: [], blocked: SAFARI_BLOCKED } + return marks || history ? { browser, profiles: [{ id: '.', name: 'Safari', dir }] } : undefined + } + // The Chromium family: `Local State` names the profiles. A browser without one has a single + // profile, in "Default" or in its own folder (as Opera keeps it). + const listed = chromiumProfiles(await readFile(join(dir, 'Local State'), 'utf8').catch(() => '')) + const profiles: FoundProfile[] = [] + for (const p of listed) { + const at = join(dir, p.folder) + if (await hasChromiumData(at)) profiles.push({ id: p.folder, name: p.name, dir: at }) + } + for (const folder of profiles.length > 0 ? [] : ['Default', '.']) { + if (!(await hasChromiumData(join(dir, folder)))) continue + profiles.push({ id: folder, name: 'Default', dir: join(dir, folder) }) + break + } + return profiles.length > 0 ? { browser, profiles } : undefined +} + +/** The browsers installed at these places, with their profiles (IM-02). Places with nothing are left out. */ +export async function findSources(places: readonly Place[]): Promise { + const out: FoundSource[] = [] + const seen = new Map() + for (const place of places) { + const found = await findAt(place).catch((err: unknown) => { + console.warn(`[import] couldn’t look for ${BROWSER_NAMES[place.browser]}:`, (err as Error).message) + return undefined + }) + if (!found) continue + const n = (seen.get(place.browser) ?? 0) + 1 + seen.set(place.browser, n) + const name = BROWSER_NAMES[place.browser] + out.push({ + id: n === 1 ? place.browser : `${place.browser}-${n}`, + name: n === 1 ? name : `${name} (${n})`, + ...found, + }) + } + return out +} + +/** Types of a profile that couldn't be read: the types step and the summary name them, with why (F3). */ +function unreadable(out: ProfileContents, source: FoundSource, types: ImportDataType[], err: unknown): void { + console.warn(`[import] couldn’t read ${source.name}’s ${types.join(' and ')}:`, (err as Error).message) + out.unreadable ??= {} + for (const t of types) out.unreadable[t] ??= unreadableReason(err) +} + +/** Reads types of a profile from one file: a held file marks them locked (IM-14); a broken one, unreadable. */ +async function part( + out: ProfileContents, + source: FoundSource, + types: ImportDataType[], + read: () => Promise, +): Promise { + try { + await read() + } catch (err) { + if (err instanceof LockedError) out.locked.push(...types) + else unreadable(out, source, types, err) + } +} + +/** One type read from a file that was copied: if it's broken, the file's other types are still read. */ +async function quietly( + out: ProfileContents, + source: FoundSource, + type: ImportDataType, + read: () => unknown, +): Promise { + try { + await read() + } catch (err) { + unreadable(out, source, [type], err) + } +} + +/** What a profile holds (IM-03): each type read from copies; types it can't provide are left out. */ +export async function inspectProfile( + source: FoundSource, + profile: FoundProfile, + scratch: Scratch, +): Promise { + const out: ProfileContents = { profileId: profile.id, name: profile.name, locked: [] } + const setBookmarks = (tree: BookmarkNode[] | undefined) => { + const count = tree ? countBookmarks(tree) : 0 + if (tree && count > 0) out.bookmarks = { count, tree } + } + const setHistory = (count: number) => { + if (count > 0) out.history = { count } + } + const file = (name: string) => join(profile.dir, name) + const history = historySql(source.browser) + if (source.browser === 'firefox') { + // Bookmarks and history share one file: one copy serves both. + await part(out, source, ['bookmarks', 'history'], () => + withDatabase(scratch, file(history.file), async (db) => { + await quietly(out, source, 'bookmarks', () => setBookmarks(readFirefoxBookmarks(db))) + await quietly(out, source, 'history', async () => setHistory(await countHistory(db, history))) + }), + ) + return out + } + if (source.browser === 'safari') { + await part(out, source, ['bookmarks'], async () => { + const copy = await scratch.take(file('Bookmarks.plist')) + if (copy) setBookmarks(safariBookmarks(parseBplist(await readFile(copy)))) + }) + } else { + await part(out, source, ['bookmarks'], async () => { + const copy = await scratch.take(file('Bookmarks')) + if (copy) setBookmarks(chromiumBookmarks(await readFile(copy, 'utf8'))) + }) + // Listed, not yet importable (IM-09). + await part(out, source, ['extensions'], async () => { + const items = await chromiumExtensions(scratch, profile.dir) + if (items.length) out.extensions = { items } + }) + } + await part(out, source, ['history'], () => + withDatabase(scratch, file(history.file), async (db) => setHistory(await countHistory(db, history))), + ) + return out +} diff --git a/src/main/import/chromium.ts b/src/main/import/chromium.ts new file mode 100644 index 00000000..54c1fade --- /dev/null +++ b/src/main/import/chromium.ts @@ -0,0 +1,71 @@ +import type { BookmarkNode } from '../../shared/commands/import' +import { buildTree, type SourceItem } from './netscape' + +/** + * The Chromium family's files (Chrome, Chromium, Brave, Edge, Vivaldi, Opera): `Local State` names + * the profiles; each profile keeps `Bookmarks` (JSON). Pure, so it is unit-tested on fixtures. + */ + +/** A profile as its browser lists it: its folder and the name the user gave it. */ +export interface ProfileEntry { + folder: string + name: string +} + +/** A folder name with no path in it (the list comes from another program's file). */ +export const isPlainFolder = (f: string) => !!f && f !== '.' && f !== '..' && !/[\\/\0]/.test(f) + +/** `Local State`'s profiles, in the browser's own order (IM-02). */ +export function chromiumProfiles(localState: string): ProfileEntry[] { + let data: { profile?: { info_cache?: unknown; profiles_order?: unknown } } | undefined + try { + data = JSON.parse(localState) + } catch { + return [] + } + const cache = data?.profile?.info_cache as Record | undefined + if (!cache || typeof cache !== 'object') return [] + const order = Array.isArray(data?.profile?.profiles_order) ? (data.profile.profiles_order as unknown[]) : [] + const folders = new Set([...order.filter((f): f is string => typeof f === 'string'), ...Object.keys(cache)]) + return [...folders] + .filter((f) => cache[f] && isPlainFolder(f)) + .map((folder) => { + const name = cache[folder]?.name + return { folder, name: typeof name === 'string' && name.trim() ? name.trim() : folder } + }) +} + +/** Chromium's three roots, by the names people know them by (IM-04, IM-05). */ +const ROOTS: [key: string, title: string][] = [ + ['bookmark_bar', 'Bookmarks bar'], + ['other', 'Other bookmarks'], + ['synced', 'Mobile bookmarks'], +] + +interface RawNode { + type?: unknown + name?: unknown + url?: unknown + children?: unknown + [key: string]: unknown +} + +/** + * A profile's `Bookmarks` as a tree: the bookmarks bar, other and mobile bookmarks become folders + * (empty ones left out), with their folders and order kept. + */ +export function chromiumBookmarks(json: string): BookmarkNode[] { + const data = JSON.parse(json) as { roots?: Record } | null + let seq = 0 + const id = () => `b${seq++}` + const list = (v: unknown): (RawNode | null)[] => (Array.isArray(v) ? v : []) + const read = (n: RawNode | null): SourceItem => { + const title = typeof n?.name === 'string' ? n.name.trim() : '' + if (n?.type === 'folder') return { title, children: list(n.children) } + if (n?.type === 'url' && typeof n.url === 'string' && n.url) return { title: title || n.url, url: n.url } + return undefined + } + return ROOTS.map(([key, title]) => ({ title, children: buildTree(list(data?.roots?.[key]?.children), 1, id, read) })) + .filter((root) => root.children.length > 0) + .map((root) => ({ id: id(), ...root })) +} diff --git a/src/main/import/copy.ts b/src/main/import/copy.ts new file mode 100644 index 00000000..2d368c10 --- /dev/null +++ b/src/main/import/copy.ts @@ -0,0 +1,151 @@ +import { rmSync } from 'node:fs' +import { copyFile, mkdir, mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { basename, join } from 'node:path' +import { DatabaseSync } from 'node:sqlite' + +/** + * Another browser's files are read from copies (IM-14): the browser may be running and holding + * them, and its SQLite files must never be opened in place. The copies go in a folder under + * Orbit's user data: the ones being read are removed at quit, and whatever a crash left is + * removed at the next startup, so no copy of someone's history outlives Orbit (D58). + */ + +/** Where scratch folders go, once Orbit has started (`setScratchFolder`); the system's temp folder before (tests). */ +let root: string | undefined +/** Startup's clearing of the folder: a new copy waits for it. */ +let cleared: Promise = Promise.resolve() +/** The scratch folders being read now. */ +const inUse = new Set() +/** How a file is copied in: e2e tests hold a copy open with it, to quit while one is (`setCopyStep`). */ +let copyStep: CopyFile = copyFile + +/** Tests (e2e): every copy from now on is made with `step`. */ +export function setCopyStep(step: CopyFile): void { + copyStep = step +} + +/** Startup: copies go in `dir`, and whatever a quit or a crash left there is removed first (D58). */ +export function setScratchFolder(dir: string): void { + root = dir + cleared = rm(dir, { recursive: true, force: true }).catch(() => {}) +} + +/** Quit: the copies being read are closed and their folders removed, before Orbit exits (D58). */ +export function removeScratch(): void { + for (const scratch of inUse) scratch.discard() + inUse.clear() +} + +/** A file the source browser holds, so it couldn't be copied: quit the browser and try again. */ +export class LockedError extends Error { + constructor(readonly file: string) { + super(`${basename(file)} is in use`) + } +} + +/** + * Why a copied file couldn't be read, as the summary says it (IM-14): the system's error reading it, + * or anything else (SQLite's "file is not a database", JSON that doesn't parse): a damaged file. + */ +export function unreadableReason(err: unknown): string { + const code = (err as { code?: unknown } | null)?.code + return typeof code === 'string' && /^E[A-Z]+$/.test(code) ? 'the file couldn’t be read' : 'the file is damaged' +} + +/** Errors that mean another program holds the file (Windows reports a sharing violation as EBUSY). */ +const LOCK_CODES = new Set(['EBUSY', 'EPERM', 'EACCES', 'ETXTBSY']) + +export type CopyFile = (from: string, to: string) => Promise + +/** A temporary folder for copies of a browser's files. */ +export class Scratch { + private n = 0 + /** The databases open on copies in it. */ + private readonly open = new Set() + /** Set at quit: its copies are gone, and nothing more is copied in. */ + private discarded = false + + constructor( + readonly dir: string, + private readonly copy: CopyFile = copyFile, + ) {} + + /** Copies a file in, with a SQLite file's log or journal; undefined when there's no such file. */ + async take(file: string): Promise { + if (this.discarded) throw new Error('Orbit is quitting') + const to = join(this.dir, `${this.n++}-${basename(file)}`) + try { + await this.copy(file, to) + } catch (err) { + const code = (err as NodeJS.ErrnoException).code ?? '' + if (code === 'ENOENT' || code === 'ENOTDIR') return undefined + if (LOCK_CODES.has(code)) throw new LockedError(file) + throw err + } + for (const extra of ['-wal', '-journal']) await this.copy(file + extra, to + extra).catch(() => {}) + return to + } + + /** Opens a copied SQLite file, closed by `close` (or at quit). */ + openDatabase(path: string): DatabaseSync { + if (this.discarded) throw new Error('Orbit is quitting') + const db = openCopy(path) + this.open.add(db) + return db + } + + close(db: DatabaseSync): void { + if (this.open.delete(db)) db.close() + } + + /** Closes its databases and removes the folder, synchronously: a read still going fails at its next step. */ + discard(): void { + this.discarded = true + for (const db of this.open) { + try { + db.close() + } catch {} + } + this.open.clear() + try { + rmSync(this.dir, { recursive: true, force: true }) + } catch {} + } +} + +/** Runs `fn` with a fresh scratch folder, removed after (or at quit). Tests pass their own way of copying. */ +export async function withScratch(fn: (scratch: Scratch) => Promise, copy?: CopyFile): Promise { + await cleared + const parent = root ?? tmpdir() + if (root) await mkdir(root, { recursive: true }) + const scratch = new Scratch(await mkdtemp(join(parent, root ? 'read-' : 'orbit-import-')), copy ?? copyStep) + inUse.add(scratch) + try { + return await fn(scratch) + } finally { + inUse.delete(scratch) + scratch.discard() + } +} + +/** Opens a copied SQLite file (Electron's own Node has `node:sqlite`: no native module). */ +export function openCopy(path: string): DatabaseSync { + return new DatabaseSync(path) +} + +/** Runs `fn` on a copy of a SQLite file; undefined when there's no such file. */ +export async function withDatabase( + scratch: Scratch, + file: string, + fn: (db: DatabaseSync) => T | Promise, +): Promise { + const copy = await scratch.take(file) + if (!copy) return undefined + const db = scratch.openDatabase(copy) + try { + return await fn(db) + } finally { + scratch.close(db) + } +} diff --git a/src/main/import/export.ts b/src/main/import/export.ts new file mode 100644 index 00000000..e741e5a6 --- /dev/null +++ b/src/main/import/export.ts @@ -0,0 +1,50 @@ +import type { ItemId, OrbitState, ProfileId } from '../../shared/state' +import { splitLeaves } from '../../shared/tree' +import type { ExportNode } from './netscape' + +/** Addresses another browser can open: web pages and files. */ +const OPENABLE = /^(https?|file|ftp):/i + +/** + * Orbit's pinned items as a bookmarks tree (IM-17): the Favorites first, as a folder, then each + * Space as a folder of its pinned section, with its folders nested as they are. Today's tabs only + * when asked, as a "Today" folder in their Space. Private windows' profiles are left out. + */ +export function pinnedExport( + state: OrbitState, + opts: { profileId?: ProfileId; includeToday?: boolean } = {}, +): ExportNode[] { + const profiles = Object.values(state.profiles).filter( + (p) => p.partition.startsWith('persist:') && (!opts.profileId || p.id === opts.profileId), + ) + const out: ExportNode[] = [] + for (const p of profiles) { + const favorites = exportItems(state, p.favorites) + const title = profiles.length > 1 ? `Favorites (${p.name})` : 'Favorites' + if (favorites.length) out.push({ title, children: favorites }) + } + for (const id of state.spaceOrder) { + const space = state.spaces[id] + if (!space || !profiles.some((p) => p.id === space.profileId)) continue + const children = exportItems(state, space.pinned) + const today = opts.includeToday ? exportItems(state, space.today) : [] + if (today.length) children.push({ title: 'Today', children: today }) + out.push({ title: space.name, children, addDate: space.createdAt }) + } + return out +} + +/** Folders keep their nesting; a split's tabs stand in its place; a tab is its pinned address. */ +function exportItems(state: OrbitState, ids: readonly ItemId[]): ExportNode[] { + return ids.flatMap((id): ExportNode[] => { + const folder = state.folders[id] + if (folder) + return [{ title: folder.name, children: exportItems(state, folder.children), addDate: folder.createdAt }] + const split = state.splits[id] + if (split) return exportItems(state, splitLeaves(split.layout)) + const tab = state.tabs[id] + const url = tab?.pinnedUrl ?? tab?.url ?? '' + if (!tab || !OPENABLE.test(url)) return [] + return [{ title: tab.customTitle || tab.pinnedTitle || tab.title || url, url, addDate: tab.createdAt }] + }) +} diff --git a/src/main/import/extensions.ts b/src/main/import/extensions.ts new file mode 100644 index 00000000..b4fd0842 --- /dev/null +++ b/src/main/import/extensions.ts @@ -0,0 +1,99 @@ +import { readFile, stat } from 'node:fs/promises' +import { join } from 'node:path' +import type { SourceExtension } from '../../shared/commands/import' +import type { Scratch } from './copy' + +/** + * A Chromium-family profile's extensions (IM-09), from its `Preferences` and `Secure Preferences`. + * They are listed, not installed: the extensions lane's install path isn't in Orbit yet, so the + * import says they can't come over yet. Nothing here ever reaches a store. + */ + +/** Install locations that are the browser's own (component) extensions: never listed. */ +const BROWSERS_OWN = new Set([5, 10]) +/** Location 1 is the store's; 4 is unpacked (developer mode). */ +const FROM_STORE = 1 +/** An extension's id: 32 letters from a to p. */ +const ID = /^[a-p]{32}$/ +/** A JSON file larger than this isn't read: far above any real Preferences, and parsed on main's thread (F4). */ +export const MAX_JSON_BYTES = 16 * 1024 * 1024 +/** Extensions looked at per profile: real profiles have a few dozen. */ +const MAX_EXTENSIONS = 1000 +/** Names that mark an ad or content blocker, for IM-09's de-duplication. */ +const BLOCKER = /\b(ad ?block\w*|adguard|ublock\w*|\w*blocker|content filter)\b/i + +interface ExtensionPrefs { + location?: unknown + from_webstore?: unknown + path?: unknown + manifest?: { name?: unknown; default_locale?: unknown } +} + +interface Prefs { + extensions?: { settings?: Record; pinned_extensions?: unknown } +} + +const isObject = (v: unknown): v is Record => !!v && typeof v === 'object' && !Array.isArray(v) + +/** Under MAX_JSON_BYTES. */ +const small = (path: string) => + stat(path).then( + (s) => s.size <= MAX_JSON_BYTES, + () => false, + ) + +/** + * A JSON object of another browser's, read from a copy; undefined when it's missing, held, broken or + * too large. Listing is best effort, and nothing of a file's text is ever logged. + */ +async function readJson(scratch: Scratch, path: string): Promise | undefined> { + if (!(await small(path))) return undefined + const copy = await scratch.take(path).catch(() => undefined) + // The copy is what is read: checked again, in case the file grew. + if (!copy || !(await small(copy))) return undefined + const value: unknown = await readFile(copy, 'utf8') + .then((text) => JSON.parse(text.replace(/^\uFEFF/, ''))) + .catch(() => undefined) + return isObject(value) ? value : undefined +} + +/** A file's extension settings, when they are an object (F4: anything else would be spread key by key). */ +function settingsOf(prefs: Prefs): Record { + const settings = isObject(prefs.extensions) ? prefs.extensions.settings : undefined + return isObject(settings) ? (settings as Record) : {} +} + +export async function chromiumExtensions(scratch: Scratch, dir: string): Promise { + const prefs = async (name: string) => ((await readJson(scratch, join(dir, name))) ?? {}) as Prefs + const [open, secure] = [await prefs('Preferences'), await prefs('Secure Preferences')] + const settings = { ...settingsOf(open), ...settingsOf(secure) } + const pinnedList = isObject(open.extensions) ? open.extensions.pinned_extensions : undefined + const pinned = new Set(Array.isArray(pinnedList) ? pinnedList.slice(0, MAX_EXTENSIONS) : []) + const out: SourceExtension[] = [] + for (const [id, s] of Object.entries(settings).slice(0, MAX_EXTENSIONS)) { + if (!ID.test(id) || !s || typeof s !== 'object' || BROWSERS_OWN.has(Number(s.location))) continue + const name = await extensionName(scratch, dir, s) + if (!name) continue + const fromStore = s.from_webstore === true || Number(s.location) === FROM_STORE + out.push({ id, name, pinned: pinned.has(id), fromStore, blocker: BLOCKER.test(name) }) + } + return out.sort((a, b) => Number(b.pinned) - Number(a.pinned) || a.name.localeCompare(b.name)) +} + +/** The name in its prefs, or in its manifest on disk; a "__MSG_x__" name from its default locale. */ +async function extensionName(scratch: Scratch, dir: string, s: ExtensionPrefs): Promise { + let manifest = s.manifest && typeof s.manifest === 'object' ? s.manifest : undefined + const folder = typeof s.path === 'string' && !s.path.includes('..') ? join(dir, 'Extensions', s.path) : undefined + if (typeof manifest?.name !== 'string' && folder) manifest = await readJson(scratch, join(folder, 'manifest.json')) + const name = typeof manifest?.name === 'string' ? manifest.name.trim() : '' + const key = /^__MSG_(\w+)__$/.exec(name)?.[1] + if (!key) return name || undefined + const locale = + typeof manifest?.default_locale === 'string' && /^[\w-]+$/.test(manifest.default_locale) + ? manifest.default_locale + : 'en' + const messages = folder ? await readJson(scratch, join(folder, '_locales', locale, 'messages.json')) : undefined + const entry = Object.entries(messages ?? {}).find(([k]) => k.toLowerCase() === key.toLowerCase())?.[1] + const message = (entry as { message?: unknown } | undefined)?.message + return typeof message === 'string' && message.trim() ? message.trim() : undefined +} diff --git a/src/main/import/files.ts b/src/main/import/files.ts new file mode 100644 index 00000000..b9d1c3b2 --- /dev/null +++ b/src/main/import/files.ts @@ -0,0 +1,161 @@ +import type { BookmarkNode, ParsedFile } from '../../shared/commands/import' +import { isLongList, MAX_FILE_BYTES, parseLinkList, TOO_LARGE } from '../../shared/commands/import' +import { fallbackTitle } from '../../shared/url' +import { + bookmark, + countBookmarks, + cutTitle, + MAX_BOOKMARKS, + MAX_DEPTH, + MAX_FOLDERS, + NO_BOOKMARKS, + readBookmarksHtml, +} from './netscape' + +/** Why a bookmark (or link) past `MAX_BOOKMARKS` was left out (IM-14): folders don't count toward it (N7). */ +export const OVER_LIMIT = 'More than 200,000 in one import' + +/** IM-13: the folder a bookmarks file goes into. */ +export const FILE_FOLDER_NAME = 'Imported bookmarks' + +/** A list of links, read once (D58): how many links it holds, its first lines, and whether it's too long to edit. */ +export interface LinksInfo { + count: number + preview: string[] + /** Too long to edit (D62 (5)): the import window shows only the count and the preview. */ + long: boolean +} + +/** Lines of a long list the import window shows, and how much of each. */ +const PREVIEW_LINES = 8 +const PREVIEW_CHARS = 200 + +/** Counts a list's links and takes its first lines (IM-12, D62 (5)). Linear in the text's size. */ +export function linksInfo(text: string): LinksInfo { + const preview: string[] = [] + for (let at = 0; at < text.length && preview.length < PREVIEW_LINES; ) { + const end = text.indexOf('\n', at) + const line = text.slice(at, Math.min(end < 0 ? text.length : end, at + PREVIEW_CHARS)) + preview.push(line.endsWith('\r') ? line.slice(0, -1) : line) + if (end < 0) break + at = end + 1 + } + return { count: parseLinkList(text).links.length, preview, long: isLongList(text) } +} + +/** A dropped or chosen file, read (IM-13): its bookmarks, its links, or why it can't be read. */ +export type FileContents = + | { kind: 'bookmarks'; name: string; tree: BookmarkNode[]; count: number; cut: number } + | { kind: 'links'; name: string; info: LinksInfo } + | Extract + +/** Reads a dropped or chosen file: a bookmarks HTML file, or a plain list of links (IM-13). */ +export function parseFile(name: string, text: string): FileContents { + if (text.length > MAX_FILE_BYTES) return { kind: 'error', name, message: TOO_LARGE } + const looksHtml = /<(!doctype|dl|dt|a)[\s>]/i.test(text) + if (/\.txt$/i.test(name) || !looksHtml) { + const info = linksInfo(text) + if (info.count > 0) return { kind: 'links', name, info } + return { kind: 'error', name, message: NO_BOOKMARKS } + } + const read = readBookmarksHtml(text) + if (!read) return { kind: 'error', name, message: NO_BOOKMARKS } + return { kind: 'bookmarks', name, tree: read.tree, count: countBookmarks(read.tree), cut: read.cut } +} + +/** IM-12: the folder a list of links goes into when the import window sends no name. */ +export const LINKS_FOLDER_NAME = 'Imported links' + +/** + * A list of links (IM-12), pasted or from a .txt file, as bookmarks in their order. Lines that + * aren't links are only counted (`lines`), and so are links past `MAX_BOOKMARKS` (`cut`). + */ +export function linkNodes(text: string): { nodes: BookmarkNode[]; lines: number; cut: number } { + const { links, skipped } = parseLinkList(text.slice(0, MAX_FILE_BYTES)) + const nodes = links.slice(0, MAX_BOOKMARKS).map((l, i) => bookmark(`l${i}`, cutTitle(l.title ?? ''), l.url)) + return { nodes, lines: skipped, cut: Math.max(0, links.length - MAX_BOOKMARKS) } +} + +/** + * What the reader worker is asked (D58): a dropped file's bytes, a chosen file's path, a pasted list's bytes, or a list + * of links to make bookmarks of (its text, or the bytes main holds). Bytes are UTF-8 and are moved to the worker, not + * copied: main's thread never turns them into text (D62 (5)). + */ +export type ReadRequest = + | { op: 'file'; name: string; bytes: Uint8Array } + | { op: 'path'; name: string; path: string } + | { op: 'list'; name: string; bytes: Uint8Array } + | { op: 'links'; list: string | Uint8Array } + +/** + * The worker's answer for a file or a pasted list: a bookmarks file's tree as JSON text; a list's count and first lines, + * with its text when it edits well, or else its bytes, moved back for main to hold (D62 (5)); or why it can't be read. + */ +export type FileRead = + | Extract + | { kind: 'links'; name: string; info: LinksInfo; text?: string; bytes?: Uint8Array } + +/** UTF-8 bytes as text, as a dropped file's `text()` reads them (a byte-order mark dropped, bad bytes replaced). */ +export function decodeText(bytes: Uint8Array): string { + return new TextDecoder().decode(bytes) +} + +/** A list of links, read: its bookmarks as JSON text in parts, which main reads one at a time (D58). */ +export interface LinksRead { + parts: string[] + lines: number + cut: number +} + +/** Bookmarks per part of a `LinksRead`: reading one part takes main a few milliseconds. */ +export const PART_SIZE = 5000 + +/** A bookmark tree sent back by the import window, checked again (addresses, size, depth). */ +export function cleanTree(nodes: unknown): { nodes: BookmarkNode[]; cut: number } { + // Iterative, so no tree is too deep for it. Folders deeper than `MAX_DEPTH`, or past `MAX_FOLDERS`, are flattened, as + // a file's are, and bookmarks past `MAX_BOOKMARKS` are counted, not kept (F1, F3, N7). Every value is visited once. + const out: BookmarkNode[] = [] + let folders = MAX_FOLDERS + let bookmarks = MAX_BOOKMARKS + let seq = 0 + let cut = 0 + const lists: { list: unknown[]; out: BookmarkNode[]; depth: number }[] = [] + const seen = new Set() + const enter = (list: unknown, into: BookmarkNode[], depth: number) => { + if (Array.isArray(list) && !seen.has(list)) { + seen.add(list) + lists.push({ list, out: into, depth }) + } + } + enter(nodes, out, 0) + // Each list is read to its end before the lists it holds, in reverse so that the first is read first. + for (let at = lists.pop(); at; at = lists.pop()) { + const held: { list: unknown; out: BookmarkNode[]; depth: number }[] = [] + for (const raw of at.list) { + const n = raw as Partial | null + const title = typeof n?.title === 'string' ? cutTitle(n.title) : '' + if (Array.isArray(n?.children)) { + if (folders <= 0 || bookmarks <= 0 || at.depth >= MAX_DEPTH) { + held.push({ list: n.children, out: at.out, depth: at.depth }) + continue + } + folders-- + const folder: BookmarkNode = { id: `c${seq++}`, title, children: [] } + at.out.push(folder) + held.push({ list: n.children, out: folder.children as BookmarkNode[], depth: at.depth + 1 }) + } else if (typeof n?.url === 'string' && n.url) { + if (bookmarks <= 0) { + cut++ + continue + } + bookmarks-- + at.out.push(bookmark(`c${seq++}`, title || fallbackTitle(n.url), n.url)) + } + } + for (let i = held.length - 1; i >= 0; i--) { + const h = held[i] as (typeof held)[number] + enter(h.list, h.out, h.depth) + } + } + return { nodes: out, cut } +} diff --git a/src/main/import/firefox.ts b/src/main/import/firefox.ts new file mode 100644 index 00000000..f247c392 --- /dev/null +++ b/src/main/import/firefox.ts @@ -0,0 +1,93 @@ +import type { DatabaseSync } from 'node:sqlite' +import type { BookmarkNode } from '../../shared/commands/import' +import { buildTree, type SourceItem } from './netscape' + +/** + * Firefox's files: `profiles.ini` lists the profiles; each keeps its bookmarks and history in + * `places.sqlite`, read from a copy (IM-14). Pure but for the queries, so it is unit-tested on fixtures. + */ + +export interface FirefoxProfile { + name: string + path: string + relative: boolean +} + +/** `profiles.ini`'s profiles: the one an install opens by default first, then in the file's order (IM-02). */ +export function firefoxProfiles(ini: string): FirefoxProfile[] { + const sections: { name: string; values: Record }[] = [] + for (const raw of ini.split(/\r?\n/)) { + const line = raw.trim() + if (!line || line.startsWith(';') || line.startsWith('#')) continue + const head = /^\[(.+)\]$/.exec(line) + if (head) sections.push({ name: head[1] as string, values: {} }) + const eq = line.indexOf('=') + const at = sections[sections.length - 1] + if (!head && eq > 0 && at) at.values[line.slice(0, eq).trim()] = line.slice(eq + 1).trim() + } + const installDefaults = new Set(sections.filter((s) => s.name.startsWith('Install')).map((s) => s.values.Default)) + const rank = (s: (typeof sections)[number]) => + installDefaults.has(s.values.Path) ? 0 : s.values.Default === '1' ? 1 : 2 + return sections + .filter((s) => /^Profile\d+$/.test(s.name) && s.values.Path) + .sort((a, b) => rank(a) - rank(b)) + .map((s) => ({ + name: s.values.Name || (s.values.Path as string), + path: s.values.Path as string, + relative: s.values.IsRelative !== '0', + })) +} + +/** A row of `moz_bookmarks` with its page's address and title. */ +export interface PlacesRow { + id: number + type: number + parent: number + position: number + title: string | null + guid: string | null + url: string | null + pageTitle: string | null +} + +const BOOKMARK_ROWS = `SELECT b.id AS id, b.type AS type, b.parent AS parent, b.position AS position, b.title AS title, + b.guid AS guid, p.url AS url, p.title AS pageTitle FROM moz_bookmarks b LEFT JOIN moz_places p ON p.id = b.fk` + +/** Firefox's roots by the names it shows; its tags aren't bookmarks to import. */ +const ROOTS: [guid: string, title: string][] = [ + ['toolbar_____', 'Bookmarks Toolbar'], + ['menu________', 'Bookmarks Menu'], + ['unfiled_____', 'Other Bookmarks'], + ['mobile______', 'Mobile Bookmarks'], +] + +/** Bookmarks as a tree: toolbar, menu, other and mobile bookmarks as folders; separators and tags left out. */ +export function firefoxBookmarks(rows: readonly PlacesRow[]): BookmarkNode[] { + const children = new Map() + for (const r of rows) { + const list = children.get(r.parent) ?? [] + list.push(r) + children.set(r.parent, list) + } + for (const list of children.values()) list.sort((a, b) => a.position - b.position) + let seq = 0 + const id = () => `f${seq++}` + // A damaged file can make a folder its own ancestor: buildTree reads each list once. + const read = (r: PlacesRow): SourceItem => { + const title = (r.title ?? '').trim() + if (r.type === 2) return { title, children: children.get(r.id) ?? [] } + if (r.type === 1 && r.url) return { title: title || (r.pageTitle ?? '').trim() || r.url, url: r.url } + return undefined + } + const byGuid = new Map(rows.map((r) => [r.guid, r])) + return ROOTS.map(([guid, title]) => { + const root = byGuid.get(guid) + return { title, children: root ? buildTree(children.get(root.id) ?? [], 1, id, read) : [] } + }) + .filter((root) => root.children.length > 0) + .map((root) => ({ id: id(), ...root })) +} + +export function readFirefoxBookmarks(db: DatabaseSync): BookmarkNode[] { + return firefoxBookmarks(db.prepare(BOOKMARK_ROWS).all() as unknown as PlacesRow[]) +} diff --git a/src/main/import/history.ts b/src/main/import/history.ts new file mode 100644 index 00000000..216c2825 --- /dev/null +++ b/src/main/import/history.ts @@ -0,0 +1,400 @@ +import { join } from 'node:path' +import type { DatabaseSync } from 'node:sqlite' +import { + type BrowserKind, + type ImportProgressItem, + type StarterSuggestion, + unreadableSkip, +} from '../../shared/commands/import' +import type { ProfileId } from '../../shared/state' +import type { HistoryService } from '../history' +import { hostOf, type ImportedPage, isRecordable, MAX_ENTRIES, MAX_VISITS_PER_ENTRY } from '../history/store' +import type { FoundProfile, FoundSource } from './browsers' +import { LockedError, type Scratch, unreadableReason, withDatabase, withScratch } from './copy' +import type { ImportJobs, Job, Task } from './jobs' + +/** + * Other browsers' history (IM-07): Chromium's `History`, Firefox's `places.sqlite` and Safari's + * `History.db`, read from copies a chunk of pages at a time with a turn of the event loop between, + * so a large history never holds up the main process. Each page brings its title, its visit and + * typed counts, and its newest visit times. It goes into the target profile's history (never tabs). + */ + +/** Rows per read and pages per addition: a few milliseconds of work each. */ +export const HISTORY_CHUNK = 1000 +/** Rows per step of a count. */ +const COUNT_CHUNK = 10_000 + +/** + * How a browser's history tables are read. Every query covers one window of rows by id, so none + * takes long however large the file. Times come out of SQL in Unix ms (Chromium's don't fit a double). + */ +export interface HistorySql { + /** The file in a profile's folder. */ + file: string + /** The table of pages, and which of its rows are pages Orbit can import. */ + table: string + where: string + /** A page's id, url, title, vc (visits), tc (typed) and last (ms). */ + columns: string + /** Visits to the pages whose ids are in (?, ?]: id, at (ms), typed (1 or 0), title. */ + visits: string +} + +/** Web pages only: a browser's own pages and scripts aren't history Orbit can open. */ +const WEB = (col: string) => `(${col} LIKE 'http:%' OR ${col} LIKE 'https:%' OR ${col} LIKE 'file:%')` + +/** Chromium's clock: microseconds since 1601. */ +const CHROMIUM_MS = (col: string) => `(${col} / 1000 - 11644473600000)` + +const CHROMIUM: HistorySql = { + file: 'History', + table: 'urls', + where: `hidden = 0 AND last_visit_time > 0 AND ${WEB('url')}`, + columns: `id, url, title, visit_count AS vc, typed_count AS tc, ${CHROMIUM_MS('last_visit_time')} AS last`, + // A frame's load (a subframe transition) isn't a page the user visited. + visits: `SELECT url AS id, ${CHROMIUM_MS('visit_time')} AS at, (transition & 255) = 1 AS typed, NULL AS title + FROM visits WHERE url > ? AND url <= ? AND (transition & 255) NOT IN (3, 4)`, +} + +const FIREFOX: HistorySql = { + file: 'places.sqlite', + table: 'moz_places', + where: `hidden = 0 AND visit_count > 0 AND ${WEB('url')}`, + columns: 'id, url, title, visit_count AS vc, typed AS tc, last_visit_date / 1000 AS last', + // Embedded and framed loads and downloads aren't pages the user visited; type 2 is typed. + visits: `SELECT place_id AS id, visit_date / 1000 AS at, visit_type = 2 AS typed, NULL AS title + FROM moz_historyvisits WHERE place_id > ? AND place_id <= ? AND visit_type NOT IN (4, 7, 8)`, +} + +const SAFARI: HistorySql = { + file: 'History.db', + table: 'history_items', + where: WEB('url'), + columns: 'id, url, NULL AS title, visit_count AS vc, 0 AS tc, NULL AS last', + // Safari's clock: seconds since 2001. It keeps titles per visit. + visits: `SELECT history_item AS id, CAST(ROUND((visit_time + 978307200) * 1000) AS INTEGER) AS at, 0 AS typed, title + FROM history_visits WHERE history_item > ? AND history_item <= ?`, +} + +export function historySql(browser: BrowserKind): HistorySql { + return browser === 'firefox' ? FIREFOX : browser === 'safari' ? SAFARI : CHROMIUM +} + +/** The table's rows by id, `size` at a time, as (after, last] windows, with a turn of the event loop after each. */ +async function* windows(db: DatabaseSync, sql: HistorySql, size: number): AsyncGenerator<[number, number]> { + const next = db.prepare(`SELECT MAX(id) AS last FROM (SELECT id FROM ${sql.table} WHERE id > ? ORDER BY id LIMIT ?)`) + let after = Number.MIN_SAFE_INTEGER + for (;;) { + const last = (next.get(after, size) as { last?: unknown } | undefined)?.last + if (typeof last !== 'number') return + yield [after, last] + after = last + await new Promise((resolve) => setImmediate(resolve)) + } +} + +/** How many pages a history holds that Orbit can import (IM-03's count), counted a window at a time. */ +export async function countHistory(db: DatabaseSync, sql: HistorySql): Promise { + const count = db.prepare(`SELECT COUNT(*) AS n FROM ${sql.table} WHERE id > ? AND id <= ? AND ${sql.where}`) + let n = 0 + for await (const [after, last] of windows(db, sql, COUNT_CHUNK)) { + n += Number((count.get(after, last) as { n?: unknown } | undefined)?.n) || 0 + } + return n +} + +interface PageRow { + id: number + url: unknown + title: unknown + vc: unknown + tc: unknown + last: unknown +} + +interface VisitRow { + id: number + at: unknown + typed: unknown + title: unknown +} + +const num = (v: unknown) => (typeof v === 'number' && Number.isFinite(v) ? v : typeof v === 'bigint' ? Number(v) : 0) +const text = (v: unknown) => (typeof v === 'string' ? v.trim() : '') + +/** A page and its visits as the history store takes them; undefined for one Orbit can't keep. */ +function toPage(row: PageRow, visits: readonly VisitRow[]): ImportedPage | undefined { + const url = text(row.url) + if (!isRecordable(url)) return undefined + const times: number[] = [] + let typed = 0 + const own = text(row.title) + let title = own + let titleAt = Number.NEGATIVE_INFINITY + for (const v of visits) { + const at = Math.round(num(v.at)) + if (at <= 0) continue + times.push(at) + if (num(v.typed)) typed++ + // Safari keeps a title per visit: the newest one names the page. + const t = own ? '' : text(v.title) + if (t && at > titleAt) { + title = t + titleAt = at + } + } + times.sort((a, b) => b - a) + // The page's last visit, should the browser have let go of that visit's row. + const last = Math.round(num(row.last)) + if (last > (times[0] ?? 0)) times.unshift(last) + if (times.length === 0) return undefined + const visitCount = Math.max(num(row.vc), times.length) + return { + url, + title, + visitCount, + typedCount: Math.min(Math.max(num(row.tc), typed), visitCount), + visits: times.slice(0, MAX_VISITS_PER_ENTRY), + } +} + +/** Reads a history a window of rows at a time, with a turn of the event loop after each. */ +export async function* historyChunks( + db: DatabaseSync, + sql: HistorySql, + size = HISTORY_CHUNK, +): AsyncGenerator { + const pages = db.prepare(`SELECT ${sql.columns} FROM ${sql.table} WHERE id > ? AND id <= ? AND ${sql.where}`) + const visits = db.prepare(sql.visits) + for await (const [after, last] of windows(db, sql, size)) { + const byId = new Map() + for (const v of visits.all(after, last) as unknown as VisitRow[]) { + const list = byId.get(v.id) + if (list) list.push(v) + else byId.set(v.id, [v]) + } + const out: ImportedPage[] = [] + for (const r of pages.all(after, last) as unknown as PageRow[]) { + const page = toPage(r, byId.get(r.id) ?? []) + if (page) out.push(page) + } + yield out + } +} + +/** + * The pages visited most recently, as many as a profile's history keeps (MAX_ENTRIES): a larger + * history is trimmed as it is read, so it never sits in memory whole. + */ +export class NewestPages { + private list: ImportedPage[] = [] + /** Each page's last visit, beside it (trimming reads numbers, not pages). */ + private lastVisits: number[] = [] + /** Pages left out for being older than the ones kept. */ + older = 0 + + constructor(private readonly max = MAX_ENTRIES) {} + + add(pages: readonly ImportedPage[]): void { + for (const p of pages) { + this.list.push(p) + this.lastVisits.push(p.visits[0] ?? 0) + } + if (this.list.length >= 2 * this.max) this.trim() + } + + get size(): number { + return Math.min(this.list.length, this.max) + } + + take(): ImportedPage[] { + this.trim() + return this.list + } + + private trim(): void { + const n = this.list.length + if (n <= this.max) return + const sorted = Float64Array.from(this.lastVisits).sort() + const from = n - this.max + const cutoff = sorted[from] as number + // As many pages visited at the cutoff time as fit. + let ties = 0 + while (from + ties < n && sorted[from + ties] === cutoff) ties++ + const list: ImportedPage[] = [] + const lastVisits: number[] = [] + for (let i = 0; i < n; i++) { + const at = this.lastVisits[i] as number + if (at < cutoff || (at === cutoff && ties-- <= 0)) continue + list.push(this.list[i] as ImportedPage) + lastVisits.push(at) + } + this.list = list + this.lastVisits = lastVisits + this.older += n - list.length + } +} + +/** + * Reads a profile's history (IM-07), handing each chunk of pages to `each`; false when it has none. + * Stops early when `stop` says so. A file the browser holds throws LockedError (IM-14). + */ +export async function readProfileHistory( + source: FoundSource, + profile: FoundProfile, + scratch: Scratch, + each: (pages: ImportedPage[]) => void, + stop: () => boolean = () => false, +): Promise { + const sql = historySql(source.browser) + const read = await withDatabase(scratch, join(profile.dir, sql.file), async (db) => { + for await (const pages of historyChunks(db, sql)) { + each(pages) + if (stop()) break + } + return true + }) + return read ?? false +} + +/** + * Starter suggestions (IM-16): the sites visited most, each as its most-visited page (its home + * page when that is a search or other query). The first half are offered as Favorites, the rest + * as pinned tabs; none is added until accepted. + */ +export function starterSuggestions(pages: readonly ImportedPage[], count = 6): StarterSuggestion[] { + const sites = new Map() + for (const p of pages) { + const host = /^https?:/i.test(p.url) ? hostOf(p.url)?.host : undefined + if (!host) continue + const site = sites.get(host) + if (!site) sites.set(host, { visits: p.visitCount, top: p }) + else { + site.visits += p.visitCount + if (p.visitCount > site.top.visitCount) site.top = p + } + } + return [...sites.entries()] + .sort(([, a], [, b]) => b.visits - a.visits || (b.top.visits[0] ?? 0) - (a.top.visits[0] ?? 0)) + .slice(0, count) + .map(([host, { top }], i) => { + const home = top.url.includes('?') ? `${hostOf(top.url)?.origin}/` : undefined + return { + url: home ?? top.url, + title: (!home && top.title) || host, + kind: i < Math.ceil(count / 2) ? 'favorite' : 'pinned', + accepted: false, + } + }) +} + +/** Where history may go: a profile that keeps it (private ones never do). */ +export type HistoryTarget = (profileId: ProfileId) => boolean + +/** + * The import's history line (IM-07, IM-14): reads the chosen profiles' history first (Cancel + * imports nothing), then adds it to each destination profile's history a chunk at a time, and + * writes it once. Pages already there gain only the visits they lacked (IM-15). + */ +export function historyTask( + jobs: Pick, + service: Pick, + source: FoundSource, + profiles: readonly FoundProfile[], + keeps: HistoryTarget, +): Task { + const item: ImportProgressItem = { type: 'history', state: 'waiting', done: 0 } + const read = new Map() + return { + item, + read: async (job: Job) => { + let locked = false + let failed = false + await withScratch(async (scratch) => { + for (const profile of profiles) { + if (job.cancelled) return + const kept = new NewestPages() + try { + const found = await readProfileHistory( + source, + profile, + scratch, + (pages) => { + kept.add(pages) + item.done += pages.length + jobs.changed(job) + }, + () => job.cancelled, + ) + if (found && kept.size > 0) read.set(profile.id, kept) + } catch (err) { + // Orbit is quitting: the copy was closed and removed (D58). + if (job.cancelled) return + // A file the browser holds asks for it to be quit (IM-14); a broken one is named in the summary (F3). + if (err instanceof LockedError) locked = true + else { + failed = true + console.warn(`[import] couldn’t read ${source.name}’s history:`, (err as Error).message) + jobs.addSkips(job, [unreadableSkip(profile.name, 'history', unreadableReason(err))]) + } + } + } + }) + if (locked) { + const types = job.status.locked?.types ?? [] + job.status.locked = { browser: source.name, types: [...new Set([...types, 'history' as const])] } + } + item.total = [...read.values()].reduce((n, kept) => n + kept.size, 0) + item.done = 0 + // Onboarding's import proposes a few of the sites visited most (IM-16). + if (job.plan.hold && item.total > 0) { + job.status.suggestions = starterSuggestions([...read.values()].flatMap((kept) => kept.take())) + } + if (item.total === 0) { + item.state = failed ? 'failed' : 'skipped' + item.note = locked ? `${source.name} is using it` : failed ? 'It couldn’t be read' : 'Nothing to import' + } + }, + add: async (job: Job) => { + // Each destination profile's pages, from every source profile sent to it. + const byProfile = new Map() + let older = 0 + let unkept = 0 + for (const [key, kept] of read) { + const dest = job.destinations.get(key) + if (!dest) continue + if (!keeps(dest.profileId)) { + unkept += kept.size + continue + } + const lists = byProfile.get(dest.profileId) ?? [] + lists.push(kept.take()) + older += kept.older + byProfile.set(dest.profileId, lists) + } + read.clear() + for (const [profileId, lists] of byProfile) { + const store = service.store(profileId) + await store.load() + for (const pages of lists) { + for (let from = 0; from < pages.length; from += HISTORY_CHUNK) { + await new Promise((resolve) => setImmediate(resolve)) + store.importPages(pages.slice(from, from + HISTORY_CHUNK)) + item.done += Math.min(HISTORY_CHUNK, pages.length - from) + jobs.changed(job) + } + } + await store.finishImport() + } + jobs.addSkips(job, [ + { + what: older === 1 ? 'older page' : 'older pages', + count: older, + reason: `History keeps the ${MAX_ENTRIES.toLocaleString('en-US')} most recent pages`, + }, + { what: unkept === 1 ? 'page' : 'pages', count: unkept, reason: 'A private window keeps no history' }, + ]) + }, + } +} diff --git a/src/main/import/index.ts b/src/main/import/index.ts index f287cd3f..6199d6dd 100644 --- a/src/main/import/index.ts +++ b/src/main/import/index.ts @@ -1,4 +1,387 @@ +import { randomUUID } from 'node:crypto' +import { copyFile, stat, writeFile } from 'node:fs/promises' +import { basename, join } from 'node:path' +import { app, BrowserWindow, dialog } from 'electron' +import { ACTIONS_BY_ID } from '../../shared/actions' +import { + type ImportDataType, + type ImportPlan, + type ImportProgressItem, + type ImportSkip, + type ImportSource, + type ImportStart, + type Inspection, + isLongList, + MAX_FILE_BYTES, + PASTED_LIST, + type ParsedFile, + type ProfileContents, + TOO_LARGE, + unreadableSkip, +} from '../../shared/commands/import' +import type { WindowId } from '../../shared/state' import type { MainContext } from '../context' +import type { HistoryService } from '../history' +import type { CallContext } from '../registry' +import { type FoundSource, findSources, inspectProfile, publicSource } from './browsers' +import { removeScratch, setCopyStep, setScratchFolder, withScratch } from './copy' +import { pinnedExport } from './export' +import { cleanTree, FILE_FOLDER_NAME, type FileRead, LINKS_FOLDER_NAME, OVER_LIMIT } from './files' +import { historyTask } from './history' +import { ImportJobs, type Task } from './jobs' +import { countBookmarks, NO_BOOKMARKS, writeBookmarksHtml } from './netscape' +import { importPlaces } from './places' +import { readFileOffMain, readLinksOffMain, stopReaders, UNREADABLE } from './reader' +import { provideImportStop } from './unplaced' +import { IMPORT_WINDOW_ID, ImportWindow } from './window' -/** Import feature (main process). Owned by its build lane; see docs/ARCHITECTURE.md. */ -export function registerImport(_ctx: MainContext): void {} +export function registerImport(ctx: MainContext, history: HistoryService): void { + const { commands, actions, store, windows } = ctx + const jobs = new ImportJobs(ctx) + // Copies of another browser's files go under user data: what a crash left is cleared now, and the + // copies being read are removed at quit, before Orbit exits (D58). This runs before the state's last write. + setScratchFolder(join(ctx.paths.userData, 'import-scratch')) + app.on('will-quit', () => { + jobs.quit() + stopReaders() + removeScratch() + }) + // Restore Data stops a running import before it replaces the sidebar (D65 (3)). + provideImportStop(() => jobs.stopForRestore()) + if (ctx.e2e) { + let resume: (() => void) | undefined + const hooks = { + /** + * The next import stops adding once `at` bookmarks are made, until `resumeAdding` or the quit: D62's and D65's + * tests act part-way. + */ + holdAddingAt: (at: number) => { + const hold = new Promise((resolve) => { + resume = resolve + }) + jobs.testPause = (made) => (made >= at ? hold : undefined) + }, + /** A held import goes on, to its end. */ + resumeAdding: () => { + jobs.testPause = undefined + resume?.() + }, + /** The list of links main holds for the import window, as its id and size (fix check 2 N5). */ + heldList: () => (held ? { id: held.id, bytes: held.bytes.byteLength } : null), + /** Each copy of another browser's file is made, then its step never finishes: only the quit can remove it. */ + holdCopying: () => { + setCopyStep(async (from, to) => { + await copyFile(from, to) + await new Promise(() => {}) + }) + }, + } + ;(globalThis as unknown as { __orbitImport: typeof hooks }).__orbitImport = hooks + } + // A list of links too long to edit (D62 (5)): main holds its bytes for the import window, which shows only its count + // and first lines. One at a time: the newest replaces it; an import started from it, the window's Clear, or the window + // closing lets it go. Main's thread never makes text of it: the bytes come from the window and go to the reader, + // moved, not copied. + let held: { id: string; bytes: Uint8Array } | undefined + /** A file or a list, read: a long list is held here, and the window gets its count and first lines (D62 (5)). */ + const present = (read: FileRead): ParsedFile => { + if (read.kind !== 'links') return read + const count = read.info.count + if (!read.bytes) return { kind: 'links', name: read.name, count, text: read.text ?? '' } + held = { id: randomUUID(), bytes: read.bytes } + return { kind: 'links', name: read.name, count, held: { id: held.id, count, preview: read.info.preview } } + } + /** A pasted list's bytes: over the cap it says so (D62 (5)); else counted once, in the reader, and held when long. */ + const checkedList = async (name: string, bytes: unknown): Promise => { + if (!(bytes instanceof Uint8Array)) return { kind: 'error', name, message: UNREADABLE } + if (bytes.byteLength > MAX_FILE_BYTES) return { kind: 'error', name, message: TOO_LARGE } + return present(await readFileOffMain({ op: 'list', name, bytes })) + } + /** Where the import window is asked to start (a paste or a drop on the sidebar), checked: a pasted list is read. */ + const checkStart = async (start: unknown): Promise => { + const s = start as Partial<{ kind: string; bytes: unknown; file: unknown }> | undefined + if (s?.kind === 'file' && s.file && typeof s.file === 'object') { + const file = s.file as ParsedFile + // A list sent as text too long to edit (only an older window would) is held all the same. + if (file.kind === 'links' && 'text' in file && typeof file.text === 'string' && isLongList(file.text)) { + return { kind: 'file', file: await checkedList(file.name, new TextEncoder().encode(file.text)) } + } + return { kind: 'file', file } + } + if (s?.kind !== 'links') return undefined + const list = await checkedList(PASTED_LIST, s.bytes) + if (list.kind === 'links' && 'held' in list) return { kind: 'links', held: list.held } + if (list.kind === 'links') return { kind: 'links', text: list.text } + return { kind: 'file', file: list } + } + const win = new ImportWindow( + ctx, + () => { + const windowId = win.browserWindow() + const spaceId = windowId ? store.state.windows[windowId]?.spaceId : store.state.spaceOrder[0] + return { + ...(spaceId ? { spaceId, profileId: store.state.spaces[spaceId]?.profileId } : {}), + job: jobs.status(), + } + }, + () => { + held = undefined + }, + ) + /** The browser window a call acts for (the import window and onboarding act for one). */ + const browserOf = (c: CallContext): WindowId | undefined => { + const id = c.windowId + return id && store.state.windows[id]?.kind === 'main' ? id : win.browserWindow() + } + + commands.register('import.open', async ({ start }, c) => { + const from = c.windowId === IMPORT_WINDOW_ID ? undefined : (c.windowId ?? windows.focusedWindowId()) + win.open(from, await checkStart(start)) + }) + commands.register('import.session', () => win.takeSession()) + + // The browsers found by the last look (IM-02). Their folders stay here; the window gets ids and names. + let found = new Map() + const discover = async () => { + const list = await findSources(importPlaces(ctx.e2e)) + found = new Map(list.map((s) => [s.id, s])) + return list + } + const sourceOf = async (id: unknown): Promise => { + const source = found.get(String(id)) ?? (await discover()).find((s) => s.id === id) + if (!source || source.blocked) throw new Error('That browser can’t be read') + return source + } + commands.register('import.sources', async (): Promise => (await discover()).map(publicSource)) + // What the last look at each profile couldn't read: the import's summary names it too (F3). + const unreadable = new Map>() + commands.register('import.inspect', async ({ sourceId, profileIds }): Promise => { + const source = await sourceOf(sourceId) + const ids = new Set(Array.isArray(profileIds) ? profileIds : []) + const profiles = await withScratch(async (scratch) => { + const out = [] + for (const p of source.profiles) if (ids.has(p.id)) out.push(await inspectProfile(source, p, scratch)) + return out + }) + for (const p of profiles) unreadable.set(`${source.id}/${p.profileId}`, p.unreadable ?? {}) + return { sourceId: source.id, browser: source.browser, name: source.name, profiles } + }) + // A file is read in a worker, never on main's thread (D58). + // Its bytes go to the worker as they came, moved, not copied (D62 (5)). + commands.register('import.parseFile', async ({ name, bytes }): Promise => { + const file = typeof name === 'string' ? basename(name) : 'file' + if (!(bytes instanceof Uint8Array)) return { kind: 'error', name: file, message: NO_BOOKMARKS } + if (bytes.byteLength > MAX_FILE_BYTES) return { kind: 'error', name: file, message: TOO_LARGE } + return present(await readFileOffMain({ op: 'file', name: file, bytes })) + }) + commands.register('import.holdLinks', ({ bytes }) => checkedList(PASTED_LIST, bytes)) + commands.register('import.letGo', ({ id }) => { + if (held?.id === id) held = undefined + }) + commands.register('import.chooseFile', async (_args, c): Promise => { + const parent = (c.windowId === IMPORT_WINDOW_ID ? win.window : undefined) ?? BrowserWindow.getFocusedWindow() + const options: Electron.OpenDialogOptions = { + title: 'Import from a File', + properties: ['openFile'], + filters: [ + { name: 'Bookmarks or links', extensions: ['html', 'htm', 'txt'] }, + { name: 'All files', extensions: ['*'] }, + ], + } + const picked = parent ? await dialog.showOpenDialog(parent, options) : await dialog.showOpenDialog(options) + const path = picked.canceled ? undefined : picked.filePaths[0] + if (!path) return null + const name = basename(path) + try { + if ((await stat(path)).size > MAX_FILE_BYTES) return { kind: 'error', name, message: TOO_LARGE } + } catch { + return { kind: 'error', name, message: UNREADABLE } + } + return present(await readFileOffMain({ op: 'path', name, path })) + }) + + commands.register('import.start', async ({ plan }, c) => { + const p = plan as ImportPlan + const item = (type: ImportProgressItem['type'], total?: number): ImportProgressItem => ({ + type, + state: 'waiting', + done: 0, + ...(total !== undefined ? { total } : {}), + }) + /** Bookmarks (or links) past the limit of one import, as a summary line (IM-14). */ + const overLimit = (count: number, [one, many] = ['bookmark', 'bookmarks']): ImportSkip => ({ + what: count === 1 ? one : many, + count, + reason: OVER_LIMIT, + }) + if (p?.kind === 'file') { + const bookmarks = item('bookmarks') + const tasks: Task[] = [ + { + item: bookmarks, + // The tree comes back from the import window: checked again, a turn after the call returns. + read: async (job) => { + await new Promise((resolve) => setImmediate(resolve)) + const { nodes, cut } = cleanTree(p.bookmarks) + bookmarks.total = countBookmarks(nodes) + job.bookmarks.set('', [{ title: '', nodes }]) + // Left out when the file was read, and now (only a window that sent more than it was given). + const fromFile = typeof p.cut === 'number' && p.cut > 0 ? Math.min(Math.floor(p.cut), 1e9) : 0 + jobs.addSkips(job, [overLimit(fromFile + cut)]) + }, + add: async (job) => jobs.addBookmarks(job, bookmarks), + }, + ] + return jobs.start(p, basename(String(p.name ?? 'file')), FILE_FOLDER_NAME, tasks, { windowId: browserOf(c) }) + } + if (p?.kind === 'links') { + // IM-12: one pinned folder, named by the user, with a pinned tab per link in the list's order. + const links = item('links') + const folderName = + (typeof p.folderName === 'string' ? p.folderName.trim().slice(0, 200) : '') || LINKS_FOLDER_NAME + const source = typeof p.name === 'string' && p.name ? basename(p.name) : 'a pasted list' + // A long list is held here (D62 (5)): the import takes it, and it's let go. + let list: string | Uint8Array = typeof p.text === 'string' ? p.text : '' + if (typeof list === 'string' && list.length > MAX_FILE_BYTES) throw new Error(TOO_LARGE) + if (typeof p.held === 'string') { + if (held?.id !== p.held) throw new Error('This list of links is gone: drop or paste it again') + list = held.bytes + held = undefined + } + const tasks: Task[] = [ + { + item: links, + read: async (job) => { + // Read in a worker (D58); main takes the answer a part at a time. + const { nodes, lines, cut } = await readLinksOffMain(list) + list = '' + links.total = countBookmarks(nodes) + job.bookmarks.set('', [{ title: '', nodes }]) + jobs.addSkips(job, [ + { what: lines === 1 ? 'line' : 'lines', count: lines, reason: 'Not a link' }, + overLimit(cut, ['link', 'links']), + ]) + }, + add: async (job) => jobs.addBookmarks(job, links), + }, + ] + return jobs.start(p, source, folderName, tasks, { windowId: browserOf(c) }) + } + if (p?.kind === 'browser') { + // IM-02 to IM-06: the chosen profiles' types, into "Imported from " (IM-05). + const source = await sourceOf(p.sourceId) + const chosen = (Array.isArray(p.profiles) ? p.profiles : []).flatMap((x) => { + const profile = source.profiles.find((sp) => sp.id === x?.profileId) + const types = new Set(Array.isArray(x?.types) ? x.types : []) + return profile ? [{ profile, types, bookmarks: x.bookmarks }] : [] + }) + const tasks: Task[] = [] + const withMarks = chosen.filter((x) => x.types.has('bookmarks')) + if (withMarks.length > 0) { + const bookmarks = item('bookmarks') + tasks.push({ + item: bookmarks, + read: async (job) => { + await new Promise((resolve) => setImmediate(resolve)) + bookmarks.total = 0 + let cut = 0 + for (const x of withMarks) { + const clean = cleanTree(x.bookmarks) + bookmarks.total += countBookmarks(clean.nodes) + cut += clean.cut + job.bookmarks.set(x.profile.id, [{ title: '', nodes: clean.nodes }]) + } + jobs.addSkips(job, [overLimit(cut)]) + }, + add: async (job) => jobs.addBookmarks(job, bookmarks), + }) + } + // IM-07: history to each destination's profile (one that keeps history: never a private one). + const withHistory = chosen.filter((x) => x.types.has('history')).map((x) => x.profile) + if (withHistory.length > 0) { + const keeps = (profileId: string) => !!store.state.profiles[profileId]?.partition.startsWith('persist:') + tasks.push(historyTask(jobs, history, source, withHistory, keeps)) + } + if (tasks.length === 0) throw new Error('Nothing to import') + const names = new Map(chosen.map((x) => [x.profile.id, x.profile.name])) + const id = jobs.start(p, source.name, `Imported from ${source.name}`, tasks, { windowId: browserOf(c) }, names) + // The types chosen that a chosen profile couldn't provide, its file damaged: named in the summary (F3). + const types = new Set(chosen.flatMap((x) => [...x.types])) + const job = jobs.get(id) + if (job) { + jobs.addSkips( + job, + chosen.flatMap((x) => + Object.entries(unreadable.get(`${source.id}/${x.profile.id}`) ?? {}).flatMap(([type, why]) => + types.has(type as ImportDataType) ? [unreadableSkip(x.profile.name, type as ImportDataType, why)] : [], + ), + ), + ) + } + return id + } + throw new Error('Unknown import') + }) + commands.register('import.status', ({ jobId }) => jobs.status(jobId)) + commands.register('import.cancel', ({ jobId }) => jobs.cancel(jobId)) + commands.register('import.commit', ({ jobId }) => jobs.commit(jobId)) + commands.register('import.close', () => win.close()) + commands.register('import.suggest', ({ jobId, url, accepted }) => jobs.suggest(jobId, url, accepted === true)) + // Export pinned items as bookmarks (IM-17): the system's save dialog, then a bookmarks file. + commands.register('import.exportPinned', async ({ profileId, includeToday }, c) => { + const parent = (c.windowId ? windows.get(c.windowId) : undefined) ?? BrowserWindow.getFocusedWindow() + const options: Electron.SaveDialogOptions = { + title: 'Export Pinned Items as Bookmarks', + defaultPath: join(app.getPath('documents'), 'Orbit pinned items.html'), + filters: [{ name: 'Bookmarks', extensions: ['html'] }], + } + const picked = parent ? await dialog.showSaveDialog(parent, options) : await dialog.showSaveDialog(options) + if (picked.canceled || !picked.filePath) return null + const nodes = pinnedExport(store.state, { profileId, includeToday: includeToday === true }) + await writeFile(picked.filePath, writeBookmarksHtml(nodes), 'utf8') + return picked.filePath + }) + + // Done (IM-14): the browser window shows the imported folder, in its Space, pinned section open. + commands.register('import.reveal', async ({ jobId }, c) => { + const job = jobs.find(jobId) + const folder = job?.status.folders[0] + const windowId = (job?.opener && store.state.windows[job.opener] ? job.opener : undefined) ?? browserOf(c) + if (c.windowId === IMPORT_WINDOW_ID) win.close() + if (!folder || !windowId || !store.state.folders[folder.folderId]) return + const run = commands.run.bind(commands) as unknown as (n: string, a: unknown, cc: CallContext) => Promise + if (store.state.windows[windowId]?.spaceId !== folder.spaceId) { + await run('spaces.switch', { spaceId: folder.spaceId }, { windowId }) + } + if (store.state.spaces[folder.spaceId]?.pinnedCollapsed) { + await run('tabs.setPinnedCollapsed', { spaceId: folder.spaceId, collapsed: false }, { windowId }) + } + const bw = windows.get(windowId) + if (bw && !bw.isDestroyed()) bw.focus() + windows.sendToWindow(windowId, 'sidebar:reveal', { id: folder.folderId }) + }) + + actions.register('app.import', ({ windowId }) => win.open(windowId)) + actions.register('app.exportPinned', async ({ windowId }) => { + await commands.run('import.exportPinned', {}, { windowId }) + }) + + // Onboarding's import is held until Start browsing commits it; ending another way adds nothing (OB-14). + store.subscribe((next, prev) => { + if (next.settings.onboarded && !prev.settings.onboarded) jobs.dropHeld() + }) + + // Keys pressed in the import window: ⌘W closes it; app and window actions act for its browser window. + actions.intercept((id, c) => { + if (c.windowId !== IMPORT_WINDOW_ID) return false + if (id === 'tab.close' || id === 'window.close') win.close() + else if (id === 'app.import') win.focus() + else if (ACTIONS_BY_ID[id]?.category === 'App' || ACTIONS_BY_ID[id]?.category === 'Window') { + const target = win.browserWindow() + const bw = target ? windows.get(target) : undefined + if (bw && !bw.isDestroyed()) bw.focus() + void actions.run(id, { windowId: target }) + } + return true + }) +} diff --git a/src/main/import/jobs.ts b/src/main/import/jobs.ts new file mode 100644 index 00000000..e6b71551 --- /dev/null +++ b/src/main/import/jobs.ts @@ -0,0 +1,535 @@ +import { monitorEventLoopDelay } from 'node:perf_hooks' +import { + type BookmarkNode, + type ImportJob, + type ImportPlan, + type ImportProgressItem, + type ImportSkip, + type ImportTarget, + STOPPED_BY_RESTORE, +} from '../../shared/commands/import' +import { DEFAULT_PROFILE_ID } from '../../shared/defaults' +import { newId } from '../../shared/ids' +import type { FolderId, ItemId, OrbitState, ProfileId, SpaceId, TabId, WindowId } from '../../shared/state' +import { containerItems, insertItem, locate, removeItem, splitLeaves } from '../../shared/tree' +import { sidebarUndo } from '../archive/inbox' +import type { MainContext } from '../context' +import type { CallContext } from '../registry' +import { detachFromSplit, reselectAway } from '../tabs/model' +import { + addUnplacedTabs, + attachFolders, + DetachedFolder, + freeFolderName, + planPlacement, + skippedLines, +} from './bookmarks' +import { hideImportTabs, releaseImportTabs } from './unplaced' + +/** Where one part of an import goes: bookmarks to the Space, the rest to its profile (IM-06). */ +export interface Destination { + spaceId: SpaceId + profileId: ProfileId +} + +/** One line of the progress view: read first (nothing added yet), then add. */ +export interface Task { + item: ImportProgressItem + read?: (job: Job) => Promise + add: (job: Job) => Promise +} + +/** A running import. Tasks read everything first, so Cancel before adding imports nothing (IM-01). */ +export interface Job { + status: ImportJob + plan: ImportPlan + opener: WindowId | undefined + cancelled: boolean + /** Bookmarks to add, per destination key (a source profile, or '' for a single target). */ + bookmarks: Map + destinations: Map + /** The root folder's base name, e.g. "Imported from Chrome" (IM-05) or "Imported bookmarks" (IM-13). */ + folderName: string + /** Source profiles' names by key: sub-folder names, and 'matching' destinations' names. */ + profileNames: Map + /** + * Tabs added that no section shows yet (D62 (3)), listed nowhere (D65 (1)): a quit, a failure or a restore part-way + * takes back exactly these. + */ + unplaced: TabId[] + /** Everything was added: only its numbers are still to come. */ + finished: boolean +} + +/** An import that has ended: only what its summary and Done need are kept (F8). */ +export interface EndedJob { + status: ImportJob + opener: WindowId | undefined +} + +export const IMPORT_UNDO_LABEL = 'Import Bookmarks' +const PROGRESS_EVERY_MS = 100 +/** + * Tabs added per store update while an import's folder is out of sight (D62 (3)). Every Orbit window takes each part + * in tens of milliseconds, so none freezes for long (N3); one update of 20,000 froze the browser window for a second. + */ +const PART_TABS = 1000 +/** After each part, main waits this many times as long as the part took it (at least MIN_PAUSE_MS): windows keep up. */ +const PAUSE_FACTOR = 2.5 +const MIN_PAUSE_MS = 20 +/** Why bookmarks made for a Space didn't go in: it was deleted while they were added. */ +export const SPACE_GONE = 'Their Space was deleted while they were added' + +/** Runs imports one after another; tells every Orbit UI how they go (`import.progress`). */ +export class ImportJobs { + /** Imports running, then ended ones as their status only (F8). */ + private jobs = new Map() + private ended = new Map() + private latest: string | undefined + private queue: Promise = Promise.resolve() + private sent = 0 + private sendTimer: NodeJS.Timeout | undefined + private release = new Map void>() + /** e2e only: awaited as bookmarks are made, with how many are, so a test can quit part-way (D62). */ + testPause: ((made: number) => Promise | undefined) | undefined + + constructor(private readonly ctx: MainContext) {} + + /** A running import; undefined once it has ended. */ + get(jobId?: string): Job | undefined { + const id = jobId ?? this.latest + return id ? this.jobs.get(id) : undefined + } + + /** An import, running or ended: its status and the window it was started from. */ + find(jobId?: string): EndedJob | undefined { + const id = jobId ?? this.latest + return id ? (this.jobs.get(id) ?? this.ended.get(id)) : undefined + } + + status(jobId?: string): ImportJob | null { + return this.find(jobId)?.status ?? null + } + + /** Starts an import made of `tasks`; returns its id. `profileNames`: the source profiles chosen, by key. */ + start( + plan: ImportPlan, + source: string, + folderName: string, + tasks: Task[], + c: CallContext, + profileNames = new Map(), + ): string { + const id = newId('x').replace(/^x-/, 'import-') + const job: Job = { + status: { id, phase: 'reading', source, items: tasks.map((t) => t.item), skipped: [], folders: [] }, + plan, + opener: c.windowId, + cancelled: false, + bookmarks: new Map(), + destinations: new Map(), + folderName, + profileNames, + unplaced: [], + finished: false, + } + this.jobs.set(id, job) + this.latest = id + const run = this.queue.then(() => this.run(job, tasks)) + this.queue = run.catch(() => undefined) + run.catch((err: unknown) => { + console.error('[import] failed:', err) + // Bookmarks are shown whole or not at all (D62): the tabs added out of sight go. + this.takeBack(job) + job.status.phase = 'failed' + job.status.error = err instanceof Error ? err.message : String(err) + this.changed(job, true) + }) + this.changed(job, true) + return id + } + + /** Cancel: only while nothing has been added (reading, or held for onboarding). */ + cancel(jobId: string): void { + const job = this.jobs.get(jobId) + if (!job || (job.status.phase !== 'reading' && job.status.phase !== 'held')) return + job.cancelled = true + job.status.phase = 'cancelled' + this.release.get(jobId)?.() + this.changed(job, true) + } + + /** Adds a held import (onboarding's Start browsing). */ + commit(jobId: string): void { + const job = this.jobs.get(jobId) + if (!job || job.cancelled) return + job.plan.hold = false + this.release.get(jobId)?.() + } + + /** Accepts or declines a held import's starter suggestion (IM-16); without a url, Dismiss all. */ + suggest(jobId: string, url: string | undefined, accepted: boolean): void { + const job = this.jobs.get(jobId) + if (!job?.status.suggestions || (job.status.phase !== 'held' && job.status.phase !== 'reading')) return + if (url === undefined) job.status.suggestions = [] + for (const s of job.status.suggestions) if (s.url === url) s.accepted = accepted + this.changed(job, true) + } + + /** + * Quit: every import stops where it is (nothing more is read, D58), and the tabs of bookmarks being added out of sight + * are taken back, so the next launch shows nothing of them (D62). Runs before the state's last write. + */ + quit(): void { + for (const job of this.jobs.values()) { + job.cancelled = true + this.release.get(job.status.id)?.() + this.takeBack(job) + } + } + + /** + * Restore Data (or its undo or redo) is about to replace the sidebar (D65 (3)): each import still reading or adding + * stops where it is. Its unfinished tabs are taken back, not archived (the user never had them), and its window says + * why it stopped. A held import (onboarding's) has added nothing, and waits. + */ + stopForRestore(): void { + for (const job of this.jobs.values()) { + const { phase } = job.status + if (job.cancelled || job.finished || (phase !== 'reading' && phase !== 'adding')) continue + job.cancelled = true + this.takeBack(job) + job.status.phase = 'failed' + job.status.error = STOPPED_BY_RESTORE + for (const item of job.status.items) if (item.state === 'running') item.state = 'failed' + this.changed(job, true) + } + } + + /** + * Takes back the tabs an import added that no section shows yet: exactly its own, by id (D62 (1)). Nothing of the + * user's can be among them. Nothing lists them (D65 (1)); one the user reached anyway goes from where it is now + * (D62 (2), D65 (2)). + */ + private takeBack(job: Job): void { + const made = job.unplaced.splice(0) + const state = this.ctx.store.state + const ids = made.filter((id) => state.tabs[id]) + if (ids.length > 0) { + const placed = placedItems(state) + const gone = new Set(ids) + this.ctx.store.update((d) => { + reselectAway(d, gone) + for (const id of ids) { + if (placed.has(id)) { + detachFromSplit(d, id) + removeItem(d, id) + } + delete d.tabs[id] + } + }) + for (const id of ids) this.ctx.tabs.destroy(id) + } + releaseImportTabs(made) + } + + /** Held imports nobody committed are dropped (onboarding ended with Skip setup, OB-14). */ + dropHeld(): void { + for (const job of this.jobs.values()) if (job.plan.hold && !job.cancelled) this.cancel(job.status.id) + } + + changed(job: Job, now = false): void { + if (job.status.id !== this.latest && !now) return + const send = () => { + this.sendTimer = undefined + this.sent = Date.now() + this.ctx.windows.broadcast('orbit:import.progress', job.status) + } + if (now || Date.now() - this.sent >= PROGRESS_EVERY_MS) { + clearTimeout(this.sendTimer) + send() + } else if (!this.sendTimer) { + this.sendTimer = setTimeout(send, PROGRESS_EVERY_MS) + } + } + + private async run(job: Job, tasks: Task[]): Promise { + const started = performance.now() + const lag = monitorEventLoopDelay({ resolution: 5 }) + lag.enable() + try { + for (const task of tasks) { + if (job.cancelled) return + if (!task.read) continue + task.item.state = 'running' + this.changed(job) + await task.read(job) + if (task.item.state === 'running') task.item.state = 'waiting' + } + if (job.cancelled) return + if (job.plan.hold) { + job.status.phase = 'held' + this.changed(job, true) + await new Promise((resolve) => this.release.set(job.status.id, resolve)) + this.release.delete(job.status.id) + if (job.cancelled) return + } + job.status.phase = 'adding' + this.changed(job, true) + await this.resolveDestinations(job) + for (const task of tasks) { + if (job.cancelled) return + if (task.item.state === 'failed' || task.item.state === 'skipped') continue + task.item.state = 'running' + this.changed(job) + await task.add(job) + if (task.item.state === 'running') task.item.state = 'done' + this.changed(job) + } + // Stopped while its last task ran (Restore Data, D65 (3)): it says so, and adds nothing more. + if (job.cancelled) return + await this.addSuggestions(job) + // Or while it added the starter suggestions: it stays stopped, never done. + if (job.cancelled) return + job.finished = true + } finally { + // One more turn of the event loop, so the histogram sees the last stretch of work end. + await new Promise((resolve) => setTimeout(resolve, 20)) + lag.disable() + job.status.stats = { ms: Math.round(performance.now() - started), longestTaskMs: Math.round(lag.max / 1e6) } + // Done once its numbers are in: whoever sees it done sees them too (fix check 2 N7). + if (job.finished) job.status.phase = 'done' + this.changed(job, true) + this.end(job) + } + } + + /** An import that has ended keeps only its status: its plan and the bookmarks read are let go (F8). */ + private end(job: Job): void { + const id = job.status.id + this.jobs.delete(id) + this.release.delete(id) + this.ended.set(id, { status: job.status, opener: job.opener }) + } + + /** The starter suggestions accepted (IM-16): Favorites of the target profile, pinned tabs of its Space, unloaded. */ + private async addSuggestions(job: Job): Promise { + const dest = [...job.destinations.values()][0] + const run = this.ctx.commands.run.bind(this.ctx.commands) as unknown as ( + n: string, + a: unknown, + c: CallContext, + ) => Promise + for (const s of job.status.suggestions ?? []) { + // Stopped meanwhile (Restore Data, D65 (3)): nothing more. + if (job.cancelled) return + if (!s.accepted || !dest) continue + const container = + s.kind === 'favorite' + ? { kind: 'favorites', profileId: dest.profileId } + : { kind: 'pinned', spaceId: dest.spaceId } + // Favorites already full: that one is left out (the tabs code says so). + await run('tabs.create', { url: s.url, container, activate: false }, { windowId: job.opener }).catch(() => {}) + } + } + + /** Makes the Spaces and profiles the plan asks for (IM-06). */ + private async resolveDestinations(job: Job): Promise { + const target: ImportTarget = job.plan.target + const state = this.ctx.store.state + const keys = new Set([...job.profileNames.keys(), ...job.bookmarks.keys(), ...job.destinations.keys()]) + if (keys.size === 0) keys.add('') + const run = this.ctx.commands.run.bind(this.ctx.commands) as unknown as ( + n: string, + a: unknown, + c: CallContext, + ) => Promise + const c: CallContext = { windowId: job.opener } + if (target.kind === 'matching') { + // In the source's order, after the last Space. + let afterSpaceId = state.spaceOrder.at(-1) + for (const key of keys) { + const name = job.profileNames.get(key) || 'Imported' + const profileId = (await run('profiles.create', { name }, c)) as ProfileId + const spaceId = (await run('spaces.create', { name, profileId, activate: false, afterSpaceId }, c)) as SpaceId + job.destinations.set(key, { spaceId, profileId }) + afterSpaceId = spaceId + } + return + } + let dest: Destination | undefined + if (target.kind === 'new-space') { + const firstSpace = state.spaces[state.spaceOrder[0] ?? ''] + const profileId = state.profiles[target.profileId] + ? target.profileId + : (firstSpace?.profileId ?? DEFAULT_PROFILE_ID) + const spaceId = (await run( + 'spaces.create', + { name: target.name.trim() || 'Imported', profileId, activate: false }, + c, + )) as SpaceId + dest = { spaceId, profileId: this.ctx.store.state.spaces[spaceId]?.profileId ?? profileId } + } else { + const space = state.spaces[target.spaceId] ?? state.spaces[state.spaceOrder[0] ?? ''] + if (!space) throw new Error('No Space to import into') + dest = { spaceId: space.id, profileId: space.profileId } + } + for (const key of keys) job.destinations.set(key, dest) + } + + /** + * Adds the bookmarks read: one pinned folder per Space (IM-05), with a sub-folder per source profile when several go + * to the same Space. Each folder is made out of sight (D62 (3)): its tabs go into the state a part at a time, where no + * section shows them, paced so every window keeps up (N3); then one store update adds the folders and shows them + * whole. Until then the sidebar shows none of it, and no list of open tabs lists its tabs (D65 (1)). One the user + * reached anyway and placed stays where they put it: its folder leaves it out (D65 (2)). One undo step takes the + * import back (D62 (4)). + */ + async addBookmarks(job: Job, item: ImportProgressItem): Promise { + const bySpace = new Map() + for (const [key, groups] of job.bookmarks) { + const dest = job.destinations.get(key) + if (!dest) continue + const list = bySpace.get(dest.spaceId) ?? [] + list.push({ key, groups }) + bySpace.set(dest.spaceId, list) + } + const now = Date.now() + const skipped = new Map() + const built: DetachedFolder[] = [] + for (const [spaceId, perProfile] of bySpace) { + const space = this.ctx.store.state.spaces[spaceId] + if (!space) continue + const asNodes = (groups: { title: string; nodes: BookmarkNode[] }[]): BookmarkNode[] => + groups.flatMap((g) => (g.title ? [{ id: g.title, title: g.title, children: g.nodes }] : g.nodes)) + const nodes: BookmarkNode[] = + perProfile.length === 1 + ? asNodes(perProfile[0]?.groups ?? []) + : perProfile.map(({ key, groups }, i) => ({ + id: `p${i}`, + title: job.profileNames.get(key) || `Profile ${i + 1}`, + children: asNodes(groups), + })) + const plan = planPlacement(job.folderName, nodes) + for (const [reason, n] of plan.skipped) skipped.set(reason, (skipped.get(reason) ?? 0) + n) + if (plan.tabs > 0) built.push(new DetachedFolder(plan, spaceId, space.profileId, now)) + } + let pause = 0 + for (const folder of built) { + while (!folder.whole && this.ctx.store.state.spaces[folder.spaceId]) { + await new Promise((resolve) => setTimeout(resolve, pause)) + if (job.cancelled) return + const started = performance.now() + const tabs = folder.build(PART_TABS) + const ids = tabs.map((t) => t.id) + // Listed nowhere from the moment they're in the state (D65 (1)). + hideImportTabs(ids) + job.unplaced.push(...ids) + this.ctx.store.update((d) => addUnplacedTabs(d, tabs)) + // Sent to the windows now, so the pause counts their share of the work. + this.ctx.store.flush() + pause = Math.max(MIN_PAUSE_MS, PAUSE_FACTOR * (performance.now() - started)) + item.done += tabs.length + this.changed(job) + await this.testPause?.(item.done) + } + } + await new Promise((resolve) => setTimeout(resolve, pause)) + if (job.cancelled) return + const state = this.ctx.store.state + // Each named now: another folder may have taken the name meanwhile (IM-15). + const names = built.map((b) => freeFolderName(state, b.spaceId, job.folderName)) + // A tab gone meanwhile leaves its folder. So does one a section holds: the user reached it anyway and placed it, and + // it stays where they put it (D65 (2)). + const placed = placedItems(state) + const kept = job.unplaced.filter((id) => placed.has(id) && state.tabs[id]).length + for (const b of built) b.keepOnly((id) => !!state.tabs[id] && !placed.has(id)) + let added: ReturnType = [] + this.ctx.store.update((d) => { + added = attachFolders(d, built, names) + }) + // Shown: from now on they are the user's (undo takes them back). A Space deleted meanwhile gets none. + const shown = new Set(added.map((a) => a.spaceId)) + const left = built.filter((b) => !shown.has(b.spaceId)) + const made = job.unplaced + job.unplaced = left.flatMap((b) => b.tabs.map((t) => t.id)) + this.takeBack(job) + releaseImportTabs(made) + const lost = left.filter((b) => !this.ctx.store.state.spaces[b.spaceId]).reduce((n, b) => n + b.total, 0) + if (lost > 0) skipped.set(SPACE_GONE, lost) + item.done = added.reduce((n, a) => n + a.tabs, kept) + for (const { folderId, spaceId } of added) job.status.folders.push({ folderId, spaceId }) + this.recordUndo(added.map((a) => a.folderId)) + job.status.skipped.push(...skippedLines(skipped, item.type === 'links' ? ['link', 'links'] : undefined)) + } + + /** Adds summary lines: those with a count of none are left out. */ + addSkips(job: Job, skips: ImportSkip[]): void { + job.status.skipped.push(...skips.filter((s) => s.count === undefined || s.count > 0)) + } + + /** ⌘Z takes an import back: its folders go, with every tab in them; ⌘⇧Z puts them back (TB-18, D62 (4)). */ + private recordUndo(rootIds: FolderId[]): void { + const stack = sidebarUndo() + if (!stack || rootIds.length === 0) return + const store = this.ctx.store + const undo = () => { + const state = store.state + const parts = rootIds.flatMap((rootId) => { + const loc = locate(state, rootId) + return loc ? [{ rootId, loc, ...subtree(state, rootId) }] : [] + }) + if (parts.length === 0) return undefined + const tabSet = new Set(parts.flatMap((p) => p.tabs.map((t) => t.id))) + store.update((d) => { + reselectAway(d, tabSet) + for (const p of parts) { + removeItem(d, p.rootId) + for (const f of p.folders) delete d.folders[f.id] + for (const t of p.tabs) delete d.tabs[t.id] + } + }) + for (const id of tabSet) this.ctx.tabs.destroy(id) + return () => { + store.update((d) => { + for (const p of parts) { + if (!containerItems(d, p.loc.container)) continue + for (const f of p.folders) d.folders[f.id] = structuredClone(f) + for (const t of p.tabs) d.tabs[t.id] = { ...structuredClone(t), asleep: true, loading: false } + insertItem(d, p.loc.container, p.loc.index, p.rootId) + } + }) + this.recordUndo(parts.map((p) => p.rootId)) + } + } + stack.push(IMPORT_UNDO_LABEL, undo, Date.now(), () => rootIds.every((id) => !store.state.folders[id])) + } +} + +/** Every item a section holds, tabs among them: Favorites, pinned and Today, in folders and split views. */ +function placedItems(state: OrbitState): Set { + const placed = new Set() + for (const p of Object.values(state.profiles)) for (const id of p.favorites) placed.add(id) + for (const sp of Object.values(state.spaces)) for (const id of [...sp.pinned, ...sp.today]) placed.add(id) + for (const f of Object.values(state.folders)) for (const id of f.children) placed.add(id) + for (const split of Object.values(state.splits)) for (const id of splitLeaves(split.layout)) placed.add(id) + return placed +} + +/** A folder's own folders and tabs, copied (for undo). */ +function subtree(state: OrbitState, rootId: FolderId) { + const folders: OrbitState['folders'][string][] = [] + const tabs: OrbitState['tabs'][string][] = [] + const walk = (id: string) => { + const f = state.folders[id] + if (f) { + folders.push(structuredClone(f)) + for (const child of f.children) walk(child) + return + } + const t = state.tabs[id] + if (t) tabs.push(structuredClone(t)) + } + walk(rootId) + return { folders, tabs } +} diff --git a/src/main/import/netscape.ts b/src/main/import/netscape.ts new file mode 100644 index 00000000..35449379 --- /dev/null +++ b/src/main/import/netscape.ts @@ -0,0 +1,283 @@ +/** + * The Netscape bookmark file (IM-13, IM-17): the HTML every browser exports and imports. + * Pure (no Electron), so it is unit-tested directly. + * + * Reading keeps folders (H3 + DL), their order, and each link's title and address; it ignores + * everything else (icons, dates, descriptions, tags). Writing produces the same format. + */ +import type { BookmarkNode } from '../../shared/commands/import' + +export const NO_BOOKMARKS = 'No bookmarks found in this file' + +/** + * Where a title ends: at its close tag or, when the file leaves that out (HTML lets it), at the next item or list. The + * search never goes past those, so each part of the file is read once (D58). + */ +const TEXT_END = /<\/?(?:a|h3|dt|dd|dl)\b/gi +/** A title is cut to this many characters (a pinned tab keeps 400). */ +export const MAX_TITLE = 1000 +/** Folders deeper than this are flattened: their items go in the deepest folder kept (F1). */ +export const MAX_DEPTH = 64 +/** Bookmarks past this many in one import are left out, and counted (IM-14). Folders don't count toward it (N7). */ +export const MAX_BOOKMARKS = 200_000 +/** Folders past this many aren't made: their items go in the folder that holds them, as a folder too deep does (F1). */ +export const MAX_FOLDERS = 20_000 + +const ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' } + +export function decodeEntities(text: string): string { + return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, code: string) => { + if (code[0] === '#') { + const n = + code[1] === 'x' || code[1] === 'X' ? Number.parseInt(code.slice(2), 16) : Number.parseInt(code.slice(1), 10) + return Number.isFinite(n) && n > 0 && n <= 0x10ffff ? String.fromCodePoint(n) : whole + } + return ENTITIES[code.toLowerCase()] ?? whole + }) +} + +/** A title's text: tags dropped, entities decoded, spaces collapsed, cut to `MAX_TITLE`. */ +export function cleanText(html: string): string { + // `[^<>]` keeps each "<" a tag can start at to one scan (D58). + const text = decodeEntities(html.replace(/<[^<>]*>/g, '')) + .replace(/\s+/g, ' ') + .trim() + return cutTitle(text) +} + +/** `text` cut to `MAX_TITLE` characters, never in the middle of a character. */ +export function cutTitle(text: string): string { + if (text.length <= MAX_TITLE) return text + const code = text.charCodeAt(MAX_TITLE - 1) + return text.slice(0, code >= 0xd800 && code <= 0xdbff ? MAX_TITLE - 1 : MAX_TITLE) +} + +function attribute(attrs: string, name: string): string | undefined { + const m = new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i').exec(attrs) + if (!m) return undefined + return decodeEntities(m[1] ?? m[2] ?? m[3] ?? '').trim() +} + +/** + * Why a bookmark's address can't be a pinned tab, or undefined when it can. Web pages and local + * files are kept; a browser's own pages, bookmarklets and saved searches are not. + */ +export function skipReason(url: string): string | undefined { + if (url.length > 4096) return 'Address too long' + const scheme = /^([a-z][a-z0-9+.-]*):/i.exec(url)?.[1]?.toLowerCase() + if (!scheme) return 'Not a web address' + if (scheme === 'http' || scheme === 'https') { + try { + return new URL(url).host ? undefined : 'Not a web address' + } catch { + return 'Not a web address' + } + } + if (scheme === 'file') return undefined + if (scheme === 'javascript') return 'Bookmarklet (a script, not a page)' + if (scheme === 'place') return 'Saved search, not a page' + if (/^(chrome|edge|brave|vivaldi|opera|about|chrome-extension|moz-extension|extension|safari-resource)$/.test(scheme)) + return 'A page only that browser can open' + return 'Not a web address' +} + +/** A bookmark node for `url`, marked when it can't be imported. */ +export function bookmark(id: string, title: string, url: string): BookmarkNode { + const reason = skipReason(url) + return reason ? { id, title, url, skip: reason } : { id, title, url } +} + +/** One item of another browser's bookmarks, as `buildTree` reads it: a folder and what it holds, or a bookmark. */ +export type SourceItem = { title: string; children: readonly T[] } | { title: string; url: string } | undefined + +/** + * Another browser's bookmarks as a tree, in their order. Iterative, so no source is too deep for it: folders deeper + * than `MAX_DEPTH` are flattened in place, their items going in the deepest folder kept (F1). `depth` is how many + * folders deep `items` already is. A list of items is read once, even when a damaged source holds it twice. + */ +export function buildTree( + items: readonly T[], + depth: number, + id: () => string, + read: (item: T) => SourceItem, +): BookmarkNode[] { + const root: BookmarkNode[] = [] + const seen = new Set([items]) + const stack: { items: readonly T[]; at: number; out: BookmarkNode[]; depth: number }[] = [ + { items, at: 0, out: root, depth }, + ] + for (let top = stack.at(-1); top; top = stack.at(-1)) { + if (top.at >= top.items.length) { + stack.pop() + continue + } + const item = read(top.items[top.at++] as T) + if (!item) continue + if ('url' in item) { + top.out.push(bookmark(id(), cutTitle(item.title) || cutTitle(item.url), item.url)) + continue + } + if (seen.has(item.children)) continue + seen.add(item.children) + if (top.depth >= MAX_DEPTH) { + stack.push({ items: item.children, at: 0, out: top.out, depth: top.depth }) + continue + } + const folder: BookmarkNode = { id: id(), title: cutTitle(item.title) || 'Untitled folder', children: [] } + top.out.push(folder) + stack.push({ items: item.children, at: 0, out: folder.children as BookmarkNode[], depth: top.depth + 1 }) + } + return root +} + +/** Calls `visit` for every node under `nodes`. Iterative: a tree from a file can be deeper than the stack (F1). */ +export function eachNode(nodes: readonly BookmarkNode[], visit: (node: BookmarkNode) => void): void { + const lists: (readonly BookmarkNode[])[] = [nodes] + for (let list = lists.pop(); list; list = lists.pop()) { + for (const node of list) { + visit(node) + if (node.children) lists.push(node.children) + } + } +} + +/** Bookmarks that will become pinned tabs (not counting skipped ones). */ +export function countBookmarks(nodes: readonly BookmarkNode[]): number { + let n = 0 + eachNode(nodes, (node) => { + if (!node.children && node.url && !node.skip) n++ + }) + return n +} + +/** Every bookmark, skipped ones included. */ +export function countAll(nodes: readonly BookmarkNode[]): number { + let n = 0 + eachNode(nodes, (node) => { + if (!node.children && node.url) n++ + }) + return n +} + +/** A bookmarks file read: its top-level items, and the bookmarks left out past `MAX_BOOKMARKS`. */ +export interface BookmarksFile { + tree: BookmarkNode[] + cut: number +} + +/** + * Reads a bookmarks HTML file. Returns the top-level items (the file's root list), or undefined + * when it holds no bookmarks: not this format, empty, or too broken to read. + */ +export function parseBookmarksHtml(html: string): BookmarkNode[] | undefined { + return readBookmarksHtml(html)?.tree +} + +/** + * Reads a bookmarks HTML file in one pass, in time linear in its size (D58): each tag and each title is read once. + * Folders deeper than `MAX_DEPTH`, or past `MAX_FOLDERS`, are flattened into the one that holds them, and bookmarks + * past `MAX_BOOKMARKS` are counted, not kept (F1, F3, N7). + */ +export function readBookmarksHtml(html: string): BookmarksFile | undefined { + if (!/]/i.test(html) && /]/i.test(html))) { + return undefined + } + const root: BookmarkNode[] = [] + /** The open lists, innermost last, and how many folders deep each is. */ + const stack: BookmarkNode[][] = [root] + const depths: number[] = [0] + /** A folder whose H3 was just read: the next DL holds its children. */ + let opened: BookmarkNode | undefined + let seq = 0 + let folders = 0 + let bookmarks = 0 + let cut = 0 + // `[^<>]` ends a tag's attributes at the next "<" too, so a "" is read once (D58). + const tag = /<(\/?)(dl|h3|a)\b([^<>]*)>/gi + for (let m = tag.exec(html); m; m = tag.exec(html)) { + const closing = m[1] === '/' + const name = (m[2] as string).toLowerCase() + const attrs = m[3] ?? '' + const level = stack[stack.length - 1] as BookmarkNode[] + const depth = depths[depths.length - 1] as number + if (name === 'dl') { + if (closing) { + if (stack.length > 1) { + stack.pop() + depths.pop() + } + } else { + // A DL right after a folder's title is that folder's list; any other DL is the file's own. + stack.push(opened?.children ?? level) + depths.push(opened ? depth + 1 : depth) + } + opened = undefined + continue + } + if (closing) continue + TEXT_END.lastIndex = tag.lastIndex + const end = TEXT_END.exec(html)?.index ?? html.length + const inner = html.slice(tag.lastIndex, end) + tag.lastIndex = end + opened = undefined + if (name === 'h3') { + // Too deep, past the folders' limit or the bookmarks': no folder, so the next DL's items go in this one's list (F1). + if (depth >= MAX_DEPTH || folders >= MAX_FOLDERS || bookmarks >= MAX_BOOKMARKS) continue + folders++ + const folder: BookmarkNode = { id: `n${seq++}`, title: cleanText(inner) || 'Untitled folder', children: [] } + level.push(folder) + opened = folder + } else { + const href = attribute(attrs, 'href') + if (!href) continue + if (bookmarks >= MAX_BOOKMARKS) cut++ + else { + bookmarks++ + level.push(bookmark(`n${seq++}`, cleanText(inner) || cutTitle(href), href)) + } + } + } + return countAll(root) > 0 ? { tree: root, cut } : undefined +} + +function escapeHtml(text: string): string { + return text.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"') +} + +/** One folder or link of an export (IM-17). */ +export interface ExportNode { + title: string + url?: string + children?: ExportNode[] + addDate?: number +} + +/** Writes a bookmarks HTML file that other browsers import (IM-17). */ +export function writeBookmarksHtml(nodes: readonly ExportNode[]): string { + const lines = [ + '', + '', + '', + 'Bookmarks', + '

Bookmarks

', + '

', + ] + const walk = (list: readonly ExportNode[], depth: number) => { + const pad = ' '.repeat(depth) + for (const node of list) { + const date = node.addDate ? ` ADD_DATE="${Math.floor(node.addDate / 1000)}"` : '' + if (node.children) { + lines.push(`${pad}

${escapeHtml(node.title)}`, `${pad}

`) + walk(node.children, depth + 1) + lines.push(`${pad}

`) + } else if (node.url) { + lines.push(`${pad}

${escapeHtml(node.title)}`) + } + } + } + walk(nodes, 1) + lines.push('

', '') + return lines.join('\n') +} diff --git a/src/main/import/places.ts b/src/main/import/places.ts new file mode 100644 index 00000000..0a31fd6c --- /dev/null +++ b/src/main/import/places.ts @@ -0,0 +1,109 @@ +import { homedir } from 'node:os' +import { isAbsolute, join, resolve } from 'node:path' +import type { BrowserKind } from '../../shared/commands/import' + +/** A folder where a browser keeps its profiles. */ +export interface Place { + browser: BrowserKind + dir: string +} + +/** Fixture places for tests: JSON `[{ "browser": "chrome", "dir": "…" }]`. */ +export const PLACES_ENV = 'ORBIT_IMPORT_PLACES' + +const KINDS = new Set([ + 'chrome', + 'chromium', + 'brave', + 'edge', + 'vivaldi', + 'opera', + 'opera-gx', + 'firefox', + 'safari', +]) + +/** Where each browser keeps its profiles on this OS (paths only: nothing is read here). */ +export function standardPlaces(platform: NodeJS.Platform, home: string, env: NodeJS.ProcessEnv): Place[] { + if (platform === 'darwin') { + const lib = join(home, 'Library', 'Application Support') + return [ + { browser: 'chrome', dir: join(lib, 'Google', 'Chrome') }, + { browser: 'chromium', dir: join(lib, 'Chromium') }, + { browser: 'brave', dir: join(lib, 'BraveSoftware', 'Brave-Browser') }, + { browser: 'edge', dir: join(lib, 'Microsoft Edge') }, + { browser: 'vivaldi', dir: join(lib, 'Vivaldi') }, + { browser: 'opera', dir: join(lib, 'com.operasoftware.Opera') }, + { browser: 'opera-gx', dir: join(lib, 'com.operasoftware.OperaGX') }, + { browser: 'firefox', dir: join(lib, 'Firefox') }, + { browser: 'safari', dir: join(home, 'Library', 'Safari') }, + ] + } + if (platform === 'win32') { + const local = env.LOCALAPPDATA || join(home, 'AppData', 'Local') + const roaming = env.APPDATA || join(home, 'AppData', 'Roaming') + return [ + { browser: 'chrome', dir: join(local, 'Google', 'Chrome', 'User Data') }, + { browser: 'chromium', dir: join(local, 'Chromium', 'User Data') }, + { browser: 'brave', dir: join(local, 'BraveSoftware', 'Brave-Browser', 'User Data') }, + { browser: 'edge', dir: join(local, 'Microsoft', 'Edge', 'User Data') }, + { browser: 'vivaldi', dir: join(local, 'Vivaldi', 'User Data') }, + { browser: 'opera', dir: join(roaming, 'Opera Software', 'Opera Stable') }, + { browser: 'opera-gx', dir: join(roaming, 'Opera Software', 'Opera GX Stable') }, + { browser: 'firefox', dir: join(roaming, 'Mozilla', 'Firefox') }, + ] + } + const config = env.XDG_CONFIG_HOME || join(home, '.config') + return [ + { browser: 'chrome', dir: join(config, 'google-chrome') }, + { browser: 'chromium', dir: join(config, 'chromium') }, + { browser: 'brave', dir: join(config, 'BraveSoftware', 'Brave-Browser') }, + { browser: 'edge', dir: join(config, 'microsoft-edge') }, + { browser: 'vivaldi', dir: join(config, 'vivaldi') }, + { browser: 'opera', dir: join(config, 'opera') }, + { browser: 'firefox', dir: join(home, '.mozilla', 'firefox') }, + // Firefox installed as a Snap keeps its profiles in the Snap's home. + { browser: 'firefox', dir: join(home, 'snap', 'firefox', 'common', '.mozilla', 'firefox') }, + ] +} + +/** Reads fixture places (relative folders resolve against the working directory). */ +export function parsePlaces(json: string): Place[] { + let list: unknown + try { + list = JSON.parse(json) + } catch { + throw new Error(`${PLACES_ENV} is not JSON`) + } + if (!Array.isArray(list)) throw new Error(`${PLACES_ENV} must be a list`) + return list.flatMap((p) => { + const browser = (p as Place)?.browser + const dir = (p as Place)?.dir + if (!KINDS.has(browser) || typeof dir !== 'string' || !dir) return [] + return [{ browser, dir: isAbsolute(dir) ? dir : resolve(dir) }] + }) +} + +/** + * The one place Orbit learns where to look for browsers (IM-02). Tests set fixture places. In a + * test run without them Orbit looks nowhere and reports the mistake as an uncaught error, so the + * test fails (the quit check, D28) instead of ever reading someone's real profiles. + */ +export function importPlaces( + e2e: boolean, + env: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform, + home: string = homedir(), + refuse: (message: string) => void = (message) => + setImmediate(() => { + throw new Error(message) + }), +): Place[] { + const fixture = env[PLACES_ENV] + if (fixture !== undefined) return parsePlaces(fixture) + if (e2e) { + refuse(`[import] ${PLACES_ENV} is not set in a test run, so no browser was searched`) + return [] + } + return standardPlaces(platform, home, env) +} diff --git a/src/main/import/reader-worker.ts b/src/main/import/reader-worker.ts new file mode 100644 index 00000000..cb85d866 --- /dev/null +++ b/src/main/import/reader-worker.ts @@ -0,0 +1,56 @@ +/** + * The import's reader, in a worker thread (D58): a dropped or chosen file, or a list of links, is read here and not + * on main's thread, so no file (up to the 64 MB limit) can stall the browser. Main starts one per read and ends it + * when the answer comes (`reader.ts`). Bytes come and go moved, not copied. Imports only pure modules. + */ +import { readFile } from 'node:fs/promises' +import { parentPort, workerData } from 'node:worker_threads' +import { MAX_FILE_BYTES, TOO_LARGE } from '../../shared/commands/import' +import { + decodeText, + type FileRead, + type LinksRead, + linkNodes, + linksInfo, + PART_SIZE, + parseFile, + type ReadRequest, +} from './files' + +/** The bytes in a buffer of their own, so that moving it to main moves nothing else. */ +function ownBytes(bytes: Uint8Array): Uint8Array { + return bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength ? bytes : new Uint8Array(bytes) +} + +async function answer(request: ReadRequest): Promise<{ result: FileRead | LinksRead; move: ArrayBuffer[] }> { + if (request.op === 'links') { + const { nodes, lines, cut } = linkNodes(typeof request.list === 'string' ? request.list : decodeText(request.list)) + const parts: string[] = [] + for (let i = 0; i < nodes.length; i += PART_SIZE) parts.push(JSON.stringify(nodes.slice(i, i + PART_SIZE))) + return { result: { parts, lines, cut }, move: [] } + } + const bytes = request.op === 'path' ? new Uint8Array(await readFile(request.path)) : request.bytes + if (bytes.byteLength > MAX_FILE_BYTES) + return { result: { kind: 'error', name: request.name, message: TOO_LARGE }, move: [] } + const text = decodeText(bytes) + // A pasted list is a list whatever it holds; a file is a bookmarks file or a list of links (IM-13). + const read = + request.op === 'list' + ? { kind: 'links' as const, name: request.name, info: linksInfo(text) } + : parseFile(request.name, text) + if (read.kind === 'links') { + // A list that edits well goes back as its text; a longer one as its bytes, which main holds (D62 (5)). + if (!read.info.long) return { result: { ...read, text }, move: [] } + const own = ownBytes(bytes) + return { result: { ...read, bytes: own }, move: [own.buffer as ArrayBuffer] } + } + if (read.kind !== 'bookmarks') return { result: read, move: [] } + // The tree as JSON text: main passes it on without reading it. + const json = JSON.stringify(read.tree) + return { result: { kind: 'bookmarks', name: read.name, json, count: read.count, cut: read.cut }, move: [] } +} + +answer(workerData as ReadRequest).then( + ({ result, move }) => parentPort?.postMessage({ result }, move), + (err: unknown) => parentPort?.postMessage({ error: err instanceof Error ? err.message : String(err) }), +) diff --git a/src/main/import/reader.ts b/src/main/import/reader.ts new file mode 100644 index 00000000..8e44088c --- /dev/null +++ b/src/main/import/reader.ts @@ -0,0 +1,78 @@ +/// +/** + * Reading a dropped or chosen file, or a list of links, in a worker thread (D58): main's thread only passes text on. + * Each read has its own worker, ended when it answers, fails or takes too long, and at quit. + */ +import type { Worker } from 'node:worker_threads' +import type { BookmarkNode } from '../../shared/commands/import' +import type { FileRead, LinksRead, ReadRequest } from './files' +import startReader from './reader-worker?nodeWorker' + +/** A read that takes longer than this is stopped (a 64 MB file takes a few seconds). */ +const TIMEOUT_MS = 60_000 +export const UNREADABLE = 'Orbit couldn’t read this file' + +const running = new Set() + +/** Bytes are moved to the worker, not copied (D62 (5)): only when they're the whole of their buffer. */ +function moved(request: ReadRequest): ArrayBuffer[] { + const bytes = 'bytes' in request ? request.bytes : request.op === 'links' ? request.list : undefined + if (!(bytes instanceof Uint8Array) || bytes.byteOffset !== 0 || bytes.byteLength !== bytes.buffer.byteLength) + return [] + return [bytes.buffer as ArrayBuffer] +} + +function read(request: ReadRequest): Promise { + return new Promise((resolve, reject) => { + const worker = startReader({ workerData: request, transferList: moved(request) }) + running.add(worker) + const end = (settle: () => void) => { + if (!running.delete(worker)) return + clearTimeout(timer) + void worker.terminate() + settle() + } + const timer = setTimeout(() => end(() => reject(new Error('Reading took too long'))), TIMEOUT_MS) + timer.unref() + worker.once('message', (m: { result?: T; error?: string }) => + end(() => (m.error === undefined ? resolve(m.result as T) : reject(new Error(m.error)))), + ) + worker.once('error', (err) => end(() => reject(err))) + worker.once('exit', () => end(() => reject(new Error('Reading stopped')))) + }) +} + +/** + * A dropped file, the file at `path` or a pasted list, read: its bookmarks (as JSON), its links, or why it can't be read. + * The bytes go to the worker: after this call they're no longer main's. + */ +export async function readFileOffMain( + request: Extract, +): Promise { + try { + return await read(request) + } catch { + return { kind: 'error', name: request.name, message: UNREADABLE } + } +} + +/** + * A list of links as bookmarks (IM-12): its text, or the bytes main held (moved to the worker). Main reads the worker's + * answer a part at a time, a turn apart. + */ +export async function readLinksOffMain( + list: string | Uint8Array, +): Promise<{ nodes: BookmarkNode[]; lines: number; cut: number }> { + const { parts, lines, cut } = await read({ op: 'links', list }) + const nodes: BookmarkNode[] = [] + for (const part of parts) { + for (const node of JSON.parse(part) as BookmarkNode[]) nodes.push(node) + await new Promise((resolve) => setImmediate(resolve)) + } + return { nodes, lines, cut } +} + +/** Quit: ends every read; each one's caller sees it stop. */ +export function stopReaders(): void { + for (const worker of running) void worker.terminate() +} diff --git a/src/main/import/safari.ts b/src/main/import/safari.ts new file mode 100644 index 00000000..eb52fddb --- /dev/null +++ b/src/main/import/safari.ts @@ -0,0 +1,131 @@ +import type { BookmarkNode } from '../../shared/commands/import' +import { bookmark, MAX_DEPTH } from './netscape' + +/** + * Safari's files (macOS only): `Bookmarks.plist`, a binary property list, and `History.db`. Reading + * them needs Full Disk Access; without it Safari is listed with the way round it (IM-02). Built + * from the formats' public descriptions and tested on fixtures; unverified against a real Safari. + */ + +/** More objects than any bookmarks file holds: a file that claims more is not read. */ +const MAX_OBJECTS = 2_000_000 + +/** Reads a binary property list ("bplist00"). Throws on anything else or anything broken. */ +export function parseBplist(buf: Buffer): unknown { + if (buf.length < 40 || buf.toString('latin1', 0, 8) !== 'bplist00') throw new Error('Not a binary property list') + const t = buf.length - 32 + const offsetSize = buf[t + 6] as number + const refSize = buf[t + 7] as number + const count = Number(buf.readBigUInt64BE(t + 8)) + const top = Number(buf.readBigUInt64BE(t + 16)) + const table = Number(buf.readBigUInt64BE(t + 24)) + if (!offsetSize || !refSize || count > MAX_OBJECTS || table + count * offsetSize > t) { + throw new Error('Broken property list') + } + const uint = (at: number, size: number) => { + if (at + size > t) throw new Error('Broken property list') + let v = 0 + for (let i = 0; i < size; i++) v = v * 256 + (buf[at + i] as number) + return v + } + let budget = MAX_OBJECTS + const read = (index: number, depth: number): unknown => { + if (index >= count || depth > MAX_DEPTH || budget-- <= 0) throw new Error('Broken property list') + const at = uint(table + index * offsetSize, offsetSize) + const marker = uint(at, 1) + const kind = marker >> 4 + const info = marker & 0xf + let start = at + 1 + const length = () => { + if (info !== 0xf) return info + const size = 1 << (uint(start, 1) & 0xf) + const n = uint(start + 1, size) + start += 1 + size + return n + } + switch (kind) { + case 0x0: + return info === 0x9 ? true : info === 0x8 ? false : null + case 0x1: + return uint(start, 1 << info) + case 0x2: + return info === 2 ? buf.readFloatBE(start) : buf.readDoubleBE(start) + case 0x3: + return new Date((buf.readDoubleBE(start) + 978_307_200) * 1000) + case 0x4: { + const n = length() + return buf.subarray(start, start + n) + } + case 0x5: { + const n = length() + return buf.toString('latin1', start, start + n) + } + case 0x6: { + const n = length() + return Buffer.from(buf.subarray(start, start + 2 * n)) + .swap16() + .toString('utf16le') + } + case 0x8: + return uint(start, info + 1) + case 0xa: { + const n = length() + return Array.from({ length: n }, (_, i) => read(uint(start + i * refSize, refSize), depth + 1)) + } + case 0xd: { + const n = length() + const out: Record = {} + for (let i = 0; i < n; i++) { + const key = read(uint(start + i * refSize, refSize), depth + 1) + out[String(key)] = read(uint(start + (n + i) * refSize, refSize), depth + 1) + } + return out + } + default: + throw new Error('Broken property list') + } + } + return read(top, 0) +} + +/** Safari's lists by the names it shows ("Favorites bar", so it isn't taken for Orbit's Favorites). */ +const ROOTS: Record = { + BookmarksBar: 'Favorites bar', + BookmarksMenu: 'Bookmarks Menu', + 'com.apple.ReadingList': 'Reading List', +} + +interface SafariNode { + WebBookmarkType?: unknown + Title?: unknown + URLString?: unknown + URIDictionary?: { title?: unknown } + Children?: unknown +} + +/** `Bookmarks.plist` as a tree: the favorites bar, bookmarks menu and reading list as folders (IM-04). */ +export function safariBookmarks(plist: unknown): BookmarkNode[] { + let seq = 0 + const walk = (list: unknown, depth: number, top: boolean): BookmarkNode[] => { + if (!Array.isArray(list) || depth > MAX_DEPTH) return [] + const out: BookmarkNode[] = [] + for (const n of list as (SafariNode | null)[]) { + const title = typeof n?.Title === 'string' ? n.Title.trim() : '' + if (n?.WebBookmarkType === 'WebBookmarkTypeList') { + const children = walk(n.Children, depth + 1, false) + if (!top || children.length > 0) { + out.push({ + id: `s${seq++}`, + title: (top ? ROOTS[title] : undefined) ?? (title || 'Untitled folder'), + children, + }) + } + } else if (n?.WebBookmarkType === 'WebBookmarkTypeLeaf' && typeof n.URLString === 'string' && n.URLString) { + const name = typeof n.URIDictionary?.title === 'string' ? n.URIDictionary.title.trim() : '' + out.push(bookmark(`s${seq++}`, name || n.URLString, n.URLString)) + } + } + return out + } + return walk((plist as SafariNode | null)?.Children, 1, true) +} diff --git a/src/main/import/unplaced.ts b/src/main/import/unplaced.ts new file mode 100644 index 00000000..42fbac66 --- /dev/null +++ b/src/main/import/unplaced.ts @@ -0,0 +1,46 @@ +import type { TabId } from '../../shared/state' + +/** + * An import's unfinished tabs (D62 (3), D65): the tabs of its bookmarks, in the state where no section shows them, until + * their folder is shown whole. Nothing may reach them meanwhile (D65 (1)): lists of open tabs (the command bar's) leave + * them out, and Restore Data stops the import before it replaces the sidebar (D65 (3)). This module holds no import + * code, so those lists and Restore Data can ask it. + */ +const unplaced = new Set() +let version = 0 +let stopImports: (() => void) | undefined + +/** + * Whether `id` is a tab an import made out of sight and hasn't shown yet. One the user reached and placed anyway is + * still one: a list asks the sections too, and shows such a tab where it is (D65 (2)). + */ +export function isUnplacedImportTab(id: TabId): boolean { + return unplaced.has(id) +} + +/** Changes whenever the set does: a cache built with it starts over. */ +export function unplacedImportVersion(): number { + return version +} + +/** The import: tabs about to go into the state out of sight. */ +export function hideImportTabs(ids: readonly TabId[]): void { + for (const id of ids) unplaced.add(id) + version++ +} + +/** The import: tabs shown at last, taken back, or kept where the user put them. */ +export function releaseImportTabs(ids: readonly TabId[]): void { + for (const id of ids) unplaced.delete(id) + version++ +} + +/** Set by the import: stops every running import, which takes back its unfinished tabs and says why (D65 (3)). */ +export function provideImportStop(stop: (() => void) | undefined): void { + stopImports = stop +} + +/** Restore Data, its undo and its redo call this before they replace the sidebar (D65 (3)). */ +export function stopImportsForRestore(): void { + stopImports?.() +} diff --git a/src/main/import/window.ts b/src/main/import/window.ts new file mode 100644 index 00000000..881235dd --- /dev/null +++ b/src/main/import/window.ts @@ -0,0 +1,139 @@ +import { BrowserWindow, nativeTheme, type Rectangle, screen } from 'electron' +import type { ImportSession, ImportStart } from '../../shared/commands/import' +import type { WindowId } from '../../shared/state' +import { NEUTRAL_THEME, paletteFor } from '../../shared/theme' +import type { MainContext } from '../context' +import { guardOwnWindow } from '../settings/ui-window' +import { loadRenderer, uiWebPreferences } from '../windows' + +/** The import window's id for IPC, shortcuts and actions (not a browser window). */ +export const IMPORT_WINDOW_ID = 'w-import' + +const SIZE = { width: 680, height: 600 } +const MIN_SIZE = { width: 560, height: 480 } +const isMac = process.platform === 'darwin' + +/** + * The import window (IM-01): one at a time, over the browser window it was opened from. It is that + * window's child but not modal, so browsing goes on while an import runs (IM-14). + */ +export class ImportWindow { + private win: BrowserWindow | undefined + private openerId: WindowId | undefined + private start: ImportStart | undefined + private nonce = 0 + + constructor( + private readonly ctx: MainContext, + private readonly session: () => Omit, + /** The window has closed. */ + private readonly closed: () => void = () => {}, + ) { + nativeTheme.on('updated', () => this.window?.setBackgroundColor(this.background())) + } + + get window(): BrowserWindow | undefined { + return this.win && !this.win.isDestroyed() ? this.win : undefined + } + + /** The browser window the import acts for: its opener while open, else the last one used. */ + browserWindow(): WindowId | undefined { + const state = this.ctx.store.state + if (this.openerId && state.windows[this.openerId] && this.ctx.windows.get(this.openerId)) return this.openerId + const ids = this.ctx.windows.ids().filter((id) => state.windows[id]?.kind === 'main') + return ids[ids.length - 1] + } + + /** What the page shows: the start it was opened with, once. */ + takeSession(): ImportSession { + const start = this.start + this.start = undefined + return { ...this.session(), ...(start ? { start } : {}), nonce: this.nonce } + } + + open(from: WindowId | undefined, start?: ImportStart): void { + const state = this.ctx.store.state + if (from && state.windows[from]?.kind === 'main') this.openerId = from + this.start = start + this.nonce++ + const win = this.window + if (win) { + this.focus() + if (start && !win.webContents.isLoading()) win.webContents.send('orbit:import.session', this.takeSession()) + return + } + this.create() + } + + focus(): void { + const win = this.window + if (!win) return + if (win.isMinimized()) win.restore() + win.show() + win.focus() + } + + close(): void { + this.window?.close() + } + + private create(): void { + const opener = this.openerId ? this.ctx.windows.get(this.openerId) : undefined + const win = new BrowserWindow({ + ...this.initialBounds(opener), + minWidth: MIN_SIZE.width, + minHeight: MIN_SIZE.height, + show: false, + title: 'Import', + fullscreenable: false, + minimizable: false, + ...(opener && !opener.isDestroyed() ? { parent: opener } : {}), + backgroundColor: this.background(), + ...(isMac + ? { titleBarStyle: 'hiddenInset' as const, trafficLightPosition: { x: 16, y: 16 } } + : { autoHideMenuBar: true }), + webPreferences: uiWebPreferences(IMPORT_WINDOW_ID, 'import'), + }) + this.win = win + const contents = win.webContents + this.ctx.windows.registerUi(contents, IMPORT_WINDOW_ID) + this.ctx.windows.attachShortcuts(contents) + guardOwnWindow(contents, 'import') + win.once('ready-to-show', () => { + if (win.isDestroyed()) return + win.show() + win.focus() + }) + win.once('closed', () => { + if (this.win === win) this.win = undefined + this.closed() + // Back to the browser window it was opened over. + const back = this.browserWindow() + const target = back ? this.ctx.windows.get(back) : undefined + if (target && !target.isDestroyed() && !this.ctx.windows.quitting) target.focus() + }) + loadRenderer(contents, 'import') + } + + /** Centred over the browser window it was opened from. */ + private initialBounds(opener: BrowserWindow | undefined): Rectangle { + const around = opener && !opener.isDestroyed() ? opener.getBounds() : undefined + const area = (around ? screen.getDisplayMatching(around) : screen.getPrimaryDisplay()).workArea + const width = Math.min(SIZE.width, area.width) + const height = Math.min(SIZE.height, area.height) + const centre = around + ? { x: around.x + around.width / 2, y: around.y + around.height / 2 } + : { x: area.x + area.width / 2, y: area.y + area.height / 2 } + const x = Math.round(Math.min(Math.max(centre.x - width / 2, area.x), area.x + area.width - width)) + const y = Math.round(Math.min(Math.max(centre.y - height / 2, area.y), area.y + area.height - height)) + return { x, y, width, height } + } + + private background(): string { + const state = this.ctx.store.state + const windowId = this.browserWindow() + const spaceId = (windowId ? state.windows[windowId]?.spaceId : undefined) ?? state.spaceOrder[0] + const theme = (spaceId ? state.spaces[spaceId]?.theme : undefined) ?? NEUTRAL_THEME + return paletteFor(theme, nativeTheme.shouldUseDarkColors).surface + } +} diff --git a/src/main/menu.ts b/src/main/menu.ts index fc3b190c..84ce7789 100644 --- a/src/main/menu.ts +++ b/src/main/menu.ts @@ -40,6 +40,8 @@ export function installMenu(ctx: MainContext): void { { type: 'separator' }, item('app.settings', 'Settings…'), item('settings.defaultBrowser', 'Set as Default Browser…'), + // The import flow (import lane, IM-01). + item('app.import', 'Import from Another Browser…'), { type: 'separator' }, { role: 'services' }, { type: 'separator' }, @@ -64,6 +66,9 @@ export function installMenu(ctx: MainContext): void { // The last ~10 closed tabs (archive lane, AR-09); rebuilt as tabs close and come back. { label: 'Recently Closed', submenu: recentlyClosedItems(ctx) }, { type: 'separator' }, + // Anti-lock-in (import lane, IM-17). + item('app.exportPinned', 'Export Pinned Items as Bookmarks…'), + { type: 'separator' }, item('page.print'), ], }, diff --git a/src/main/settings/index.ts b/src/main/settings/index.ts index f381ed2e..77173352 100644 --- a/src/main/settings/index.ts +++ b/src/main/settings/index.ts @@ -75,7 +75,11 @@ export function registerSettings(ctx: MainContext): void { if (!commands.has('import.open')) return false const windowId = settings.browserWindow() ?? windows.open() windows.get(windowId)?.focus() - const run = commands.run as unknown as (name: string, args: unknown, c: CallContext) => Promise + const run = commands.run.bind(commands) as unknown as ( + name: string, + args: unknown, + c: CallContext, + ) => Promise await run('import.open', {}, { windowId }) return true }) diff --git a/src/renderer/src/import/BookmarkTree.tsx b/src/renderer/src/import/BookmarkTree.tsx new file mode 100644 index 00000000..02ae325b --- /dev/null +++ b/src/renderer/src/import/BookmarkTree.tsx @@ -0,0 +1,149 @@ +import { ChevronRight, Folder } from 'lucide-react' +import { memo, useMemo, useState } from 'react' +import type { BookmarkNode } from '../../../shared/commands/import' +import { leafKey, leafKeys, type TreeRoot } from './tree' + +/** A top-level folder holding more bookmarks than this starts closed (F5). */ +const OPEN_UP_TO = 1000 + +function hostOf(url: string): string { + try { + return new URL(url).host.replace(/^www\./, '') + } catch { + return url + } +} + +/** + * A source's bookmark tree with checkboxes (IM-04). A folder's box sets every bookmark in it, and + * shows mixed when only some are chosen. Folders below the top level start closed, and so does one + * holding over 1,000 bookmarks (as the sidebar does), so a big tree draws only what is opened. The + * list isn't virtualized: an opened big folder, or a long top-level list, draws every row (F5). + */ +export function BookmarkTree({ + roots, + selected, + onChange, +}: { + roots: TreeRoot[] + selected: ReadonlySet + onChange: (next: Set) => void +}) { + const leaves = useMemo(() => new Map(), []) + const leavesOf = (root: string, node: BookmarkNode) => { + let list = leaves.get(node) + if (!list) { + list = leafKeys(root, node.children ?? []) + leaves.set(node, list) + } + return list + } + const setMany = (keys: string[], on: boolean) => { + const next = new Set(selected) + for (const k of keys) { + if (on) next.add(k) + else next.delete(k) + } + onChange(next) + } + const ctx: RowContext = { selected, leavesOf, setMany } + return ( +

+ {roots.map((root) => + root.title ? ( + + ) : ( + root.nodes.map((n) => ) + ), + )} +
+ ) +} + +interface RowContext { + selected: ReadonlySet + leavesOf: (root: string, node: BookmarkNode) => string[] + setMany: (keys: string[], on: boolean) => void +} + +function Row({ root, node, depth, ctx }: { root: string; node: BookmarkNode; depth: number; ctx: RowContext }) { + if (node.children) return + if (!node.url) return null + const key = leafKey(root, node) + const on = !node.skip && ctx.selected.has(key) + return ( + + ) +} + +const FolderRow = memo(function FolderRow({ + root, + node, + depth, + ctx, +}: { + root: string + node: BookmarkNode + depth: number + ctx: RowContext +}) { + const keys = ctx.leavesOf(root, node) + const [open, setOpen] = useState(() => depth === 0 && keys.length <= OPEN_UP_TO) + let chosen = 0 + for (const k of keys) if (ctx.selected.has(k)) chosen++ + const all = keys.length > 0 && chosen === keys.length + const some = chosen > 0 && !all + return ( +
+
+ + { + if (el) el.indeterminate = some + }} + aria-label={`Import ${node.title}`} + onChange={() => ctx.setMany(keys, !all)} + /> +
+ {open && node.children?.map((n) => )} +
+ ) +}) diff --git a/src/renderer/src/import/ImportApp.tsx b/src/renderer/src/import/ImportApp.tsx new file mode 100644 index 00000000..5be13e00 --- /dev/null +++ b/src/renderer/src/import/ImportApp.tsx @@ -0,0 +1,50 @@ +import { useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' +import type { ImportSession } from '../../../shared/commands/import' +import { NEUTRAL_THEME } from '../../../shared/theme' +import { invoke, isMac } from '../api' +import { useOrbit } from '../store' +import { cachedPalette, paletteVars, prefersReducedMotion, useDark } from '../theme' +import { ImportFlow } from './ImportFlow' + +/** + * The import window (IM-01): the import flow in the colours of the Space it was opened from. Main + * sends a new session when it is asked again with a start (a dropped file, a paste in the sidebar). + */ +export function ImportApp() { + const [session, setSession] = useState() + useEffect(() => { + let latest = -1 + const apply = (s: ImportSession) => { + if (s.nonce < latest) return + latest = s.nonce + setSession(s) + } + void invoke('import.session', {}).then(apply) + return window.orbit.on('import.session', (payload) => apply(payload as ImportSession)) + }, []) + const spaceId = useOrbit((s) => session?.spaceId ?? s.spaceOrder[0]) + const theme = useOrbit((s) => (spaceId ? s.spaces[spaceId]?.theme : undefined)) ?? NEUTRAL_THEME + const dark = useDark() + const palette = cachedPalette(theme, dark) + const style = useMemo(() => paletteVars(palette), [palette]) + useThemeFade(dark) + return ( +
+ {isMac &&
} + {session && void invoke('import.close', {})} />} +
+ ) +} + +function useThemeFade(dark: boolean): void { + const last = useRef(dark) + useLayoutEffect(() => { + if (last.current === dark) return + last.current = dark + if (prefersReducedMotion()) return + const root = document.documentElement + root.dataset.themeFade = '' + const t = setTimeout(() => delete root.dataset.themeFade, 260) + return () => clearTimeout(t) + }, [dark]) +} diff --git a/src/renderer/src/import/ImportFlow.tsx b/src/renderer/src/import/ImportFlow.tsx new file mode 100644 index 00000000..48d4f9f7 --- /dev/null +++ b/src/renderer/src/import/ImportFlow.tsx @@ -0,0 +1,864 @@ +import { ArrowRight, ClipboardList, FileUp, HelpCircle } from 'lucide-react' +import { type ReactNode, useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { + type BookmarkNode, + type HeldLinks, + type ImportDataType, + type ImportJob, + type ImportPlan, + type ImportSession, + type ImportSource, + type ImportStart, + type ImportTarget, + type Inspection, + isLongList, + listBytes, + MAX_FILE_BYTES, + PASTED_LIST, + type ParsedFile, + parseLinkList, + TOO_LARGE, +} from '../../../shared/commands/import' +import { invoke } from '../api' +import { useOrbit } from '../store' +import { BookmarkTree } from './BookmarkTree' +import { MOVING_HEADING, MOVING_INTRO, MOVING_ROUTES, PASTE_DEFAULT_NAME } from './copy' +import { Progress } from './Progress' +import { SourcePicker, TypesPicker } from './sources' +import { filterTree, leafKeys, type TreeRoot } from './tree' +import './import.css' + +type Route = 'browser' | 'file' | 'links' +type StepId = 'source' | 'profiles' | 'types' | 'bookmarks' | 'links' | 'target' | 'progress' + +const STEP_TITLES: Record = { + source: 'Bring your bookmarks along', + profiles: 'Choose profiles', + types: 'What to import', + bookmarks: 'Choose bookmarks', + links: 'Paste a list of links', + target: 'Where should it go?', + progress: 'Importing', +} + +/** A bookmarks file, read (IM-13): its tree, and its bookmarks left out past the limit of one import. */ +interface FileTree { + name: string + tree: BookmarkNode[] + cut: number +} + +const UNREADABLE = 'Orbit couldn’t read this file' + +/** The links in a list short enough to edit (D62 (5)): quick to count again as it's edited. */ +const countLinks = (text: string) => parseLinkList(text).links.length + +/** A list of links to import: its text and, when known, its count; or a long one main holds (D62 (5)). */ +type LinkList = { text: string; count?: number } | { held: HeldLinks } + +interface Target { + spaceId: string + profileId: string + newName: string + matching: boolean +} + +/** + * The import flow (IM-01…IM-06, IM-12, IM-13): Back, Next and Cancel through the steps a route + * needs. Nothing is imported before the last step's Import, and Cancel imports nothing. The import + * window shows it, and so does onboarding's import step (`embedded`, OB-03), where the import is + * held until onboarding ends. + */ +export function ImportFlow({ + session, + embedded, + onClose, + onStarted, +}: { + session: ImportSession | undefined + embedded?: boolean + /** Cancel (the window closes; onboarding moves on: "Do this later"). */ + onClose: () => void + /** Embedded: the import was started (held). */ + onStarted?: (jobId: string) => void +}) { + const [step, setStep] = useState('source') + const [route, setRoute] = useState() + const [sources, setSources] = useState() + const [sourceId, setSourceId] = useState() + const [profileIds, setProfileIds] = useState([]) + const [inspection, setInspection] = useState() + const [types, setTypes] = useState([]) + const [file, setFile] = useState() + const [fileError, setFileError] = useState() + const [linksText, setLinksText] = useState('') + /** A list too long to edit (D62 (5)): main holds its text; the window has its count and first lines only. */ + const [held, setHeld] = useState() + /** The list's links, counted once by the reader; an edit counts again (the text edits well, so that's quick). */ + const [linkCount, setLinkCount] = useState(0) + /** The .txt file the links came from (IM-13), if they weren't pasted. */ + const [linksFile, setLinksFile] = useState() + const [folderName, setFolderName] = useState(PASTE_DEFAULT_NAME) + const [selected, setSelected] = useState>(new Set()) + const [target, setTarget] = useState() + const [job, setJob] = useState(null) + const [busy, setBusy] = useState(false) + const bodyRef = useRef(null) + + const source = sources?.find((s) => s.id === sourceId) + const roots: TreeRoot[] = useMemo(() => { + if (route === 'file') return file ? [{ key: '', nodes: file.tree }] : [] + if (route !== 'browser' || !inspection || inspection === 'loading') return [] + const withMarks = inspection.profiles.filter((p) => p.bookmarks) + return withMarks.map((p) => ({ + key: p.profileId, + ...(withMarks.length > 1 ? { title: p.name } : {}), + nodes: p.bookmarks?.tree ?? [], + })) + }, [route, file, inspection]) + const allKeys = useMemo(() => roots.flatMap((r) => leafKeys(r.key, r.nodes)), [roots]) + // Each new tree is drawn afresh: which folders are open isn't carried over from another file's. + const treeSeq = useRef({ roots, n: 0 }) + if (treeSeq.current.roots !== roots) treeSeq.current = { roots, n: treeSeq.current.n + 1 } + + const steps = useMemo((): StepId[] => { + if (route === 'file') return ['source', 'bookmarks', 'target', 'progress'] + if (route === 'links') return ['source', 'links', 'target', 'progress'] + const list: StepId[] = ['source'] + if (source && source.profiles.length > 1) list.push('profiles') + list.push('types') + if (types.includes('bookmarks')) list.push('bookmarks') + list.push('target', 'progress') + return list + }, [route, source, types]) + + const go = useCallback((to: StepId) => { + setStep(to) + requestAnimationFrame(() => bodyRef.current?.querySelector('h1')?.focus({ preventScroll: true })) + }, []) + + // A new tree (a file, a profile's bookmarks) starts with everything chosen (IM-04). + useEffect(() => setSelected(new Set(allKeys)), [allKeys]) + + // Browsers on this computer (IM-02). + useEffect(() => { + void invoke('import.sources', {}).then(setSources) + }, []) + + const showList = (list: LinkList) => { + if ('held' in list) { + setHeld(list.held) + setLinksText('') + setLinkCount(list.held.count) + } else { + setHeld(undefined) + setLinksText(list.text) + setLinkCount(list.count ?? countLinks(list.text)) + } + } + + /** A dropped or chosen file (IM-13): its bookmarks to choose from, its links, or why it can't be read. */ + const openFile = (parsed: ParsedFile) => { + setFileError(undefined) + if (parsed.kind === 'error') { + setFileError(`${parsed.name}: ${parsed.message}`) + setRoute(undefined) + setStep('source') + return + } + if (parsed.kind === 'links') { + setRoute('links') + showList('held' in parsed ? { held: parsed.held } : { text: parsed.text, count: parsed.count }) + setLinksFile(parsed.name || undefined) + go('links') + return + } + setRoute('file') + // The tree comes as JSON text, which main passed on without reading (D58). + setFile({ name: parsed.name, tree: JSON.parse(parsed.json) as BookmarkNode[], cut: parsed.cut }) + go('bookmarks') + } + + // Where the window was asked to start (a dropped file, a paste in the sidebar), else a running import. + // biome-ignore lint/correctness/useExhaustiveDependencies: once per opening (the flow is keyed by it) + useEffect(() => { + const start: ImportStart | undefined = session?.start + if (start?.kind === 'links') { + setRoute('links') + showList(start) + go('links') + } else if (start?.kind === 'file') { + openFile(start.file) + } else if (!embedded && (session?.job?.phase === 'reading' || session?.job?.phase === 'adding')) { + setJob(session.job) + setStep('progress') + } + }, []) + + useEffect( + () => + window.orbit.on('import.progress', (status) => { + setJob((current) => (current && current.id === (status as ImportJob).id ? (status as ImportJob) : current)) + }), + [], + ) + + /** Reads a file, saying so when that fails (F1). */ + const readFile = async (name: string, read: () => Promise) => { + let parsed: ParsedFile | null + try { + parsed = await read() + } catch { + parsed = { kind: 'error', name, message: UNREADABLE } + } + if (parsed) openFile(parsed) + } + + const chooseFile = () => readFile('file', () => invoke('import.chooseFile', {})) + + // Inspect the chosen profiles when they change (IM-03). + const inspect = useCallback(async (id: string, profiles: string[]) => { + setInspection('loading') + const result = await invoke('import.inspect', { sourceId: id, profileIds: profiles }) + setInspection(result) + const offered = new Set() + for (const p of result.profiles) { + if (p.bookmarks) offered.add('bookmarks') + if (p.history) offered.add('history') + } + // Everything the source offers starts checked, except cookies (P2, off by default, IM-10). + setTypes([...offered]) + }, []) + + const pickSource = (id: string) => { + const s = sources?.find((x) => x.id === id) + if (!s || s.blocked) return + setRoute('browser') + setSourceId(id) + const firstProfile = s.profiles[0]?.id + setProfileIds(firstProfile ? [firstProfile] : []) + setInspection(undefined) + } + + const index = steps.indexOf(step) + + /** The list edited: one grown too long to edit (a paste, a drop) goes to main, which holds it (D62 (5)). */ + const editLinks = (text: string) => { + if (isLongList(text)) { + void holdLinks(text) + return + } + setHeld(undefined) + setLinksText(text) + setLinkCount(countLinks(text)) + } + const holdLinks = async (text: string) => { + let parsed: ParsedFile + try { + // Over the cap, in bytes as main counts it, it says so as a file over it does, and main gets none of it (D62 (5)). + const bytes = listBytes(text) + parsed = bytes + ? await invoke('import.holdLinks', { bytes }) + : { kind: 'error', name: PASTED_LIST, message: TOO_LARGE } + } catch { + parsed = { kind: 'error', name: PASTED_LIST, message: UNREADABLE } + } + if (parsed.kind === 'error') openFile(parsed) + else if (parsed.kind === 'links') { + showList('held' in parsed ? { held: parsed.held } : { text: parsed.text, count: parsed.count }) + } + } + + const canNext = (() => { + switch (step) { + case 'source': + return route === 'browser' && !!source && !source.blocked + case 'profiles': + return profileIds.length > 0 + case 'types': + return types.length > 0 && inspection !== 'loading' && !!inspection + case 'bookmarks': + return selected.size > 0 || (route === 'browser' && types.length > 1) + case 'links': + return linkCount > 0 && folderName.trim().length > 0 + case 'target': + return !!target && (target.spaceId !== 'new' || target.newName.trim().length > 0) && !busy + default: + return false + } + })() + + const next = async () => { + if (!canNext) return + const to = steps[index + 1] + if (step === 'source' && route === 'browser' && sourceId && source && source.profiles.length <= 1) { + void inspect(sourceId, profileIds) + } + if (step === 'profiles' && sourceId) void inspect(sourceId, profileIds) + if (step === 'target') { + await start() + return + } + if (to) go(to) + } + + const back = () => { + const to = steps[index - 1] + if (!to) return + if (to === 'source') { + setRoute(undefined) + setFile(undefined) + } + go(to) + } + + const plan = (): ImportPlan | undefined => { + if (!target) return undefined + const t: ImportTarget = target.matching + ? { kind: 'matching' } + : target.spaceId === 'new' + ? { kind: 'new-space', name: target.newName.trim(), profileId: target.profileId } + : { kind: 'space', spaceId: target.spaceId } + const hold = embedded ? { hold: true } : {} + if (route === 'file' && file) { + const bookmarks = filterTree(file.tree, '', selected) + return { + kind: 'file', + name: file.name, + bookmarks, + ...(file.cut > 0 ? { cut: file.cut } : {}), + target: t, + ...hold, + } + } + if (route === 'links') { + const name = linksFile ? { name: linksFile } : {} + const list = held ? { held: held.id } : { text: linksText } + return { kind: 'links', folderName: folderName.trim(), ...list, ...name, target: t, ...hold } + } + if (route === 'browser' && sourceId && inspection && inspection !== 'loading') { + return { + kind: 'browser', + sourceId, + profiles: inspection.profiles.map((p) => ({ + profileId: p.profileId, + types: types.filter((ty) => (ty !== 'bookmarks' || !!p.bookmarks) && (ty !== 'history' || !!p.history)), + ...(types.includes('bookmarks') && p.bookmarks + ? { bookmarks: filterTree(p.bookmarks.tree, p.profileId, selected) } + : {}), + })), + target: t, + ...hold, + } + } + return undefined + } + + const start = async () => { + const p = plan() + if (!p || busy) return + setBusy(true) + try { + const jobId = await invoke('import.start', { plan: p }) + if (embedded) { + onStarted?.(jobId) + return + } + setJob(await invoke('import.status', { jobId })) + go('progress') + } finally { + setBusy(false) + } + } + + const cancel = () => { + if (job && (job.phase === 'reading' || job.phase === 'held')) void invoke('import.cancel', { jobId: job.id }) + onClose() + } + + // Dropping a bookmarks file or a list of links on the flow starts that route (IM-13). + const onDrop = async (e: React.DragEvent) => { + const f = e.dataTransfer.files[0] + if (!f || step === 'progress') return + e.preventDefault() + // A file over the limit isn't read at all (F2). + if (f.size > MAX_FILE_BYTES) { + openFile({ kind: 'error', name: f.name, message: TOO_LARGE }) + return + } + // Its bytes as they are: the reader makes text of them, off main's thread (D58, D62 (5)). + await readFile(f.name, async () => + invoke('import.parseFile', { name: f.name, bytes: new Uint8Array(await f.arrayBuffer()) }), + ) + } + + // Enter is Next and Escape is Back (the window: Escape on the first step cancels). In onboarding the + // first step leaves both keys to onboarding, so Enter moves on ("Do this later") as on every step. + const onKeyDown = (e: React.KeyboardEvent) => { + if (e.defaultPrevented || e.nativeEvent.isComposing || e.metaKey || e.ctrlKey || e.altKey || e.shiftKey) return + const el = e.target instanceof Element ? e.target : null + // Enter in a text field (the folder's name) is Next too; not on a button, a checkbox or in the list of links. + if (e.key === 'Enter' && !el?.closest('button, a, select, textarea, [role="radio"], input[type="checkbox"]')) { + if (step === 'source' && !canNext) return + e.preventDefault() + if (step !== 'progress') void next() + } else if (e.key === 'Escape' && !(el instanceof HTMLSelectElement)) { + if (index > 0 && step !== 'progress') { + e.preventDefault() + back() + } else if (!embedded) { + e.preventDefault() + if (step === 'progress') onClose() + else cancel() + } + } + } + + const nextLabel = step === 'target' ? 'Import' : 'Next' + const title = step === 'source' && embedded ? 'Bring your bookmarks along' : STEP_TITLES[step] + const running = job && (job.phase === 'reading' || job.phase === 'adding') + + return ( +
{ + if (e.dataTransfer.types.includes('Files') && step !== 'progress') { + e.preventDefault() + e.dataTransfer.dropEffect = 'copy' + } + }} + onDrop={(e) => void onDrop(e)} + > +
+

+ {title} +

+ {step === 'source' && ( + <> +

+ Copy bookmarks and history from another browser on this computer, or bring them from a file. +

+ void next()} /> +
+ } + title="From a file…" + text="A bookmarks file (.html) or a list of links (.txt). You can also drop it here." + testId="import-from-file" + onClick={() => void chooseFile()} + /> + } + title="Paste a list of links" + text="One address per line, or “Title — address”." + testId="import-paste" + onClick={() => { + setRoute('links') + setLinksFile(undefined) + go('links') + }} + /> +
+ {fileError && ( +

+ {fileError} +

+ )} + + + )} + {step === 'profiles' && source && } + {step === 'types' && ( + sourceId && void inspect(sourceId, profileIds)} + /> + )} + {step === 'bookmarks' && ( + <> +
+ + {selected.size.toLocaleString()} of {allKeys.length.toLocaleString()} bookmarks selected + + + +
+ + + )} + {step === 'links' && ( + { + // Main lets go of its copy too (fix check 2 N5). + if (held) void invoke('import.letGo', { id: held.id }) + editLinks('') + }} + name={folderName} + onName={setFolderName} + count={linkCount} + /> + )} + {step === 'target' && ( + + )} + {step === 'progress' && job && } +
+
+ {step === 'progress' ? ( + <> + {running && job?.phase === 'reading' ? ( + + ) : ( + + )} + {job?.phase === 'done' ? ( + + ) : ( + + )} + + ) : ( + <> + +
+ {index > 0 && ( + + )} + {/* In onboarding, "Do this later" weighs the same as Next (OB-03). */} + +
+ + )} +
+
+ ) +} + +export function Button({ + children, + onClick, + primary, + disabled, + testId, +}: { + children: ReactNode + onClick: () => void + primary?: boolean + disabled?: boolean + testId?: string +}) { + return ( + + ) +} + +function OtherWay({ + icon, + title, + text, + testId, + onClick, +}: { + icon: ReactNode + title: string + text: string + testId: string + onClick: () => void +}) { + return ( + + ) +} + +/** "Coming from …?": the routes in plain words (IM-12). Nothing is read from that browser. */ +function MovingHelp() { + return ( +
+ + +

{MOVING_INTRO}

+
    + {MOVING_ROUTES.map((r) => ( +
  • + {r.title}. {r.text} +
    • + ))} +
+
+ ) +} + +function ProfilesStep({ + source, + chosen, + onChange, +}: { + source: ImportSource + chosen: string[] + onChange: (ids: string[]) => void +}) { + return ( + <> +

{source.name} has several profiles. Choose one or more.

+
+ {source.profiles.map((p) => ( + + ))} +
+ + ) +} + +function LinksStep({ + text, + held, + onText, + onClear, + name, + onName, + count, +}: { + text: string + held: HeldLinks | undefined + onText: (t: string) => void + onClear: () => void + name: string + onName: (n: string) => void + count: number +}) { + /** What the field would hold with `added` put in place of its selection. */ + const withAdded = (field: HTMLTextAreaElement, added: string) => + field.value.slice(0, field.selectionStart) + added + field.value.slice(field.selectionEnd) + return ( + <> +

+ Paste links copied from another browser: one address per line, or “Title — address”. Each becomes a pinned tab, + in this order, in one new folder. +

+ {held ? ( + // Too long to edit (D62 (5)): its first lines, read-only; main holds the rest. +
+ 
+            {held.preview.join('\n')}
+
+

+ This list is too long to edit here, so it’s imported as it is.{' '} + +

+
+ ) : ( +