diff --git a/forge-web/app/layout.tsx b/forge-web/app/layout.tsx index 70df4d7a8..b63743390 100644 --- a/forge-web/app/layout.tsx +++ b/forge-web/app/layout.tsx @@ -15,7 +15,12 @@ export const metadata: Metadata = { // inline bootstrap scripts. // - frame-ancestors is ignored in a CSP; the host must send it as a header (GitHub // Pages cannot; see docs/guides/identity-and-keys.md). -// - connect-src https:/wss:: DAPI endpoints + IPFS/S3/HTTPS pack backends. +// - connect-src https:/wss:: DAPI endpoints + IPFS/S3/HTTPS pack backends. Plain http to this +// machine is allowed so the storage settings can reach the user's OWN local node (a kubo +// RPC API, a MinIO endpoint), which is how kubo ships. It cannot be narrowed to one page: a +// static export has one CSP, and several combine as an intersection. It is not a read +// path: readers fetch only public https URLs (`lib/net.ts` `externalFetchUrls`), and nothing +// loopback or private is ever recorded on chain (`fitManifestUris`, `publishProblem`). // - worker-src blob:: materialization / search / pack workers run off-main-thread. const CSP = [ "default-src 'self'", @@ -23,7 +28,7 @@ const CSP = [ "style-src 'self' 'unsafe-inline'", "img-src 'self' data: https: blob:", "font-src 'self'", - "connect-src 'self' https: wss:", + "connect-src 'self' https: wss: http://127.0.0.1:* http://localhost:*", "worker-src 'self' blob:", "child-src 'self' blob:", "frame-ancestors 'none'", diff --git a/forge-web/app/settings/page.tsx b/forge-web/app/settings/page.tsx index 4c9f75dfe..89c56b84a 100644 --- a/forge-web/app/settings/page.tsx +++ b/forge-web/app/settings/page.tsx @@ -83,6 +83,16 @@ export default function SettingsPage(): JSX.Element { +
+

Storage

+

+ Your buckets and IPFS nodes for browser pushes, tested from this page, with their keys encrypted in this browser. +

+ + Storage settings → + +
+

This browser's key

diff --git a/forge-web/app/settings/storage/page.tsx b/forge-web/app/settings/storage/page.tsx new file mode 100644 index 000000000..72799e88b --- /dev/null +++ b/forge-web/app/settings/storage/page.tsx @@ -0,0 +1,46 @@ +'use client' + +/** `/settings/storage` — bring your own storage (`ux-dx-spec.md` §3.1). */ + +import Link from 'next/link' +import { HardDrive, MonitorSmartphone } from 'lucide-react' +import { AppShell } from '@/components/app-shell' +import { Button } from '@/components/ui/button' +import { EmptyState } from '@/components/ui/states' +import { StorageWizard } from '@/components/storage/storage-wizard' +import { useAuth } from '@/contexts/auth-context' +import { useUiStore } from '@/hooks/use-ui-store' + +export default function StorageSettingsPage(): JSX.Element { + const { identity } = useAuth() + const openLogin = useUiStore((s) => s.openLogin) + + return ( + +
+
+

+ Settings / Storage +

+

Storage

+

+ We host nothing. Packs go to a bucket or IPFS node you own; Platform keeps only the signed manifest and refs (about 0.0003 DASH a push). Readers check every byte against its on-chain hash, so your storage provides availability, never trust. +

+
+

+ Use a desktop browser for this step: it needs a few long fields and a copy-paste from your provider’s console. +

+ {identity ? ( + + ) : ( + openLogin()}>Sign in} + /> + )} +
+
+ ) +} diff --git a/forge-web/components/repo/settings-content.tsx b/forge-web/components/repo/settings-content.tsx index e6f776c67..2f7a8e1c8 100644 --- a/forge-web/components/repo/settings-content.tsx +++ b/forge-web/components/repo/settings-content.tsx @@ -13,7 +13,7 @@ */ import { useState } from 'react' -import { Fingerprint, ShieldPlus, Snowflake, UserCog } from 'lucide-react' +import { Fingerprint, HardDrive, ShieldPlus, Snowflake, UserCog } from 'lucide-react' import type { RepoHome } from '@/lib/view' import type { Collaborator, V1RepoRef, V2RepoRef } from '@/lib/repo' import { @@ -41,6 +41,7 @@ import { Button } from '@/components/ui/button' import { Field, Input } from '@/components/ui/input' import { ConfirmDialog } from '@/components/confirm-dialog' import { ErrorState, LoadingBlock } from '@/components/ui/states' +import { RepoStoragePolicy } from '@/components/storage/repo-storage-policy' type Role = 'write' | 'maintain' type Action = { kind: 'grant' | 'suspend' | 'revoke'; member: string; role: Role } @@ -266,6 +267,10 @@ function V2Settings({ home, repo }: { home: RepoHome; repo: V2RepoRef }): JSX.El
+
}> + +
+
}> {members.loading ? ( diff --git a/forge-web/components/storage/copy-block.tsx b/forge-web/components/storage/copy-block.tsx new file mode 100644 index 000000000..c635faf18 --- /dev/null +++ b/forge-web/components/storage/copy-block.tsx @@ -0,0 +1,33 @@ +'use client' + +import { useState } from 'react' +import { Check, Copy } from 'lucide-react' + +/** A multi-line copyable block (a CORS policy, a few shell lines). */ +export function CopyBlock({ text, label }: { text: string; label: string }): JSX.Element { + const [copied, setCopied] = useState(false) + const copy = async (): Promise => { + try { + await navigator.clipboard.writeText(text) + setCopied(true) + setTimeout(() => setCopied(false), 1200) + } catch { + /* insecure context: the text is still selectable */ + } + } + return ( +
+
+        {text}
+      
+ +
+ ) +} diff --git a/forge-web/components/storage/cost-card.tsx b/forge-web/components/storage/cost-card.tsx new file mode 100644 index 000000000..182109af3 --- /dev/null +++ b/forge-web/components/storage/cost-card.tsx @@ -0,0 +1,56 @@ +/** + * The cost comparison card of the storage wizard (`ux-dx-spec.md` §3.1 step 5), always + * visible: what a repo costs with its packs in your bucket versus on Platform. Platform fees + * are the measured per-push manifest + ref costs; bucket prices are the providers' public + * list prices, rounded, and say so. + */ + +import { CREDITS_PER_DASH, estimateChunkCredits } from '@/lib/sdk/cost' +import { dashToUsd, formatDash } from '@/lib/view/format' + +const MIB = 1024 * 1024 +/** Manifest + ref update per push, measured on moutai (≈ 0.0003 DASH). */ +const PER_PUSH_DASH = 0.0003 + +export function CostCard(): JSX.Element { + const platform50 = estimateChunkCredits(50 * MIB) / CREDITS_PER_DASH + const perMib = estimateChunkCredits(MIB) / CREDITS_PER_DASH + const pushes = PER_PUSH_DASH * 10 + const rows: [string, string, string, string][] = [ + ['Your R2 bucket', `manifest + refs ≈ ${formatDash(PER_PUSH_DASH)} DASH / push`, '≈ $0.001', `≈ ${formatDash(pushes)} DASH ≈ ${dashToUsd(pushes)} + $0.00`], + ['Your B2 / S3 bucket', 'same', '≈ $0.0003 / $0.001', `≈ ${formatDash(pushes)} DASH + < $0.01`], + ['Dash Platform', `${formatDash(perMib)} DASH / MiB, permanent`, '0', `≈ ${formatDash(platform50)} DASH ≈ ${dashToUsd(platform50)} first upload, then ${formatDash(perMib)} DASH per pushed MiB`], + ] + return ( +
+

+ What it costs +

+
+ + + + + + + + + + + {rows.map(([where, once, monthly, example]) => ( + + + + + + + ))} + +
WhereOne-time (Platform fees)Monthly50 MiB repo, 10 pushes/month
{where}{once}{monthly}{example}
+
+

+ Platform figures are estimates from measured fees; bucket prices are the providers’ list prices, rounded. USD is indicative. +

+
+ ) +} diff --git a/forge-web/components/storage/profile-form.tsx b/forge-web/components/storage/profile-form.tsx new file mode 100644 index 000000000..8962bfb2d --- /dev/null +++ b/forge-web/components/storage/profile-form.tsx @@ -0,0 +1,226 @@ +'use client' + +/** + * The per-provider form of the storage wizard (`ux-dx-spec.md` §3.1 step 2): the fields the + * guide names, each with a "where to find this" hint. Secret fields carry a lock and say where + * they go. Values live in component state only until saved into the vault. + * + * Editing a saved profile never puts its stored secrets back into the page: secret fields start + * empty, and an empty field keeps the stored value. Autocomplete is off everywhere, so a + * password manager neither offers to generate a value nor saves a bucket secret to a synced + * store. + */ + +import { useId, useState } from 'react' +import { AlertTriangle, ExternalLink, Lock } from 'lucide-react' +import { + profileProblem, + providerPreset, + type ProfilePublic, + type ProfileSecrets, + type ProviderId, + type StorageProfile, +} from '@/lib/storage' +import { Input } from '@/components/ui/input' + +const SECRET_NOTE = 'Stored encrypted in this browser only. Never sent to Forge (there is no Forge server) and never written on-chain.' +const KEEP_NOTE = 'Leave empty to keep the stored value.' + +type Values = Record +const SECRET_KEYS: readonly (keyof ProfileSecrets)[] = ['accessKeyId', 'secretAccessKey', 'sessionToken', 'apiAuth', 'pinningToken'] + +/** Providers whose suggested profile name is the provider id itself (the rest get `-main`). */ +const BARE_NAMES: readonly ProviderId[] = ['aws', 'minio', 'kubo', 'pinning'] + +function initialValues(provider: ProviderId, existing: StorageProfile | null): Values { + const preset = providerPreset(provider) + const empty: Values = + preset.kind === 's3' + ? { endpoint: '', region: 'us-east-1', bucket: '', pathStyle: true, publicUrl: '', prefix: '', accessKeyId: '', secretAccessKey: '', sessionToken: '' } + : { api: '', gateway: '', publicGateway: '', pinningEndpoint: '', apiAuth: '', pinningToken: '' } + if (existing) { + // Public settings only; secrets stay out of the DOM. + const out: Values = { ...empty, name: existing.name } + for (const [k, v] of Object.entries(existing.settings)) { + if (typeof v === 'string' || typeof v === 'boolean') out[k] = v + } + return out + } + const defaults: Values = {} + for (const [k, v] of Object.entries(preset.defaults)) if (v !== undefined) defaults[k] = v + return { name: BARE_NAMES.includes(provider) ? provider : `${provider}-main`, ...empty, ...defaults } +} + +const str = (v: string | boolean | undefined): string => (typeof v === 'string' ? v.trim() : '') +/** A URL field, trimmed and without trailing slashes. */ +const url = (v: string | boolean | undefined): string => str(v).replace(/\/+$/, '') + +/** + * The profile the form's values describe. A secret field left empty while editing keeps the + * stored value (`kept`). + */ +export function profileFromValues(provider: ProviderId, v: Values, kept: ProfileSecrets = {}): StorageProfile { + const preset = providerPreset(provider) + const secrets: Record = {} + for (const k of SECRET_KEYS) { + const value = str(v[k]) || kept[k] + if (value) secrets[k] = value + } + let settings: ProfilePublic + if (preset.kind === 's3') { + settings = { + kind: 's3', + provider: provider as 'r2' | 'b2' | 'aws' | 'minio', + endpoint: url(v['endpoint']), + region: str(v['region']) || 'us-east-1', + bucket: str(v['bucket']), + pathStyle: v['pathStyle'] !== false, + publicUrl: url(v['publicUrl']), + prefix: str(v['prefix']), + } + const { accessKeyId, secretAccessKey, sessionToken } = secrets + return { name: str(v['name']), settings, secrets: { ...(accessKeyId ? { accessKeyId } : {}), ...(secretAccessKey ? { secretAccessKey } : {}), ...(sessionToken ? { sessionToken } : {}) } } + } + if (preset.kind === 'platform') return { name: 'platform', settings: { kind: 'platform', provider: 'platform' }, secrets: {} } + settings = { + kind: preset.kind === 'ipfs-pinning-service' ? 'ipfs-pinning-service' : 'ipfs-kubo', + provider: provider as 'kubo' | 'pinning', + api: url(v['api']), + gateway: url(v['gateway']), + publicGateway: url(v['publicGateway']), + pinningEndpoint: preset.kind === 'ipfs-pinning-service' ? url(v['pinningEndpoint']) : '', + } + const { apiAuth, pinningToken } = secrets + return { name: str(v['name']), settings, secrets: { ...(apiAuth ? { apiAuth } : {}), ...(pinningToken && preset.kind === 'ipfs-pinning-service' ? { pinningToken } : {}) } } +} + +/** The draft a form starts from (so an edit can be tested before any field changes). */ +export function initialDraft(provider: ProviderId, existing: StorageProfile | null): StorageProfile { + return profileFromValues(provider, initialValues(provider, existing), existing?.secrets ?? {}) +} + +interface FieldSpec { + readonly key: string + readonly label: string + readonly secret?: boolean + readonly optional?: boolean + readonly placeholder?: string +} + +function fieldsFor(provider: ProviderId): FieldSpec[] { + const kind = providerPreset(provider).kind + if (kind === 's3') { + return [ + { key: 'name', label: 'Profile name' }, + { key: 'endpoint', label: 'S3 endpoint' }, + { key: 'region', label: 'Region' }, + { key: 'bucket', label: 'Bucket' }, + { key: 'publicUrl', label: 'Public URL', placeholder: 'https://pub-….r2.dev' }, + { key: 'prefix', label: 'Key prefix', optional: true, placeholder: 'forge/' }, + { key: 'accessKeyId', label: 'Access key id', secret: true }, + { key: 'secretAccessKey', label: 'Secret access key', secret: true }, + { key: 'sessionToken', label: 'Session token (temporary credentials)', secret: true, optional: true }, + ] + } + if (kind === 'platform') return [] + const f: FieldSpec[] = [ + { key: 'name', label: 'Profile name' }, + { key: 'api', label: 'kubo RPC API' }, + { key: 'gateway', label: 'Gateway (verification re-read)', optional: true }, + { key: 'publicGateway', label: 'Public gateway (recorded for browsers)', optional: true }, + { key: 'apiAuth', label: 'API Authorization header', secret: true, optional: true, placeholder: 'Bearer …' }, + ] + if (kind === 'ipfs-pinning-service') { + f.push({ key: 'pinningEndpoint', label: 'Pinning Service API URL' }, { key: 'pinningToken', label: 'Pinning access token', secret: true }) + } + return f +} + +export function ProfileForm({ + provider, + existing, + onChange, +}: { + provider: ProviderId + existing: StorageProfile | null + /** The profile the form currently describes, with why it is not usable yet (or null). */ + onChange: (profile: StorageProfile, problem: string | null) => void +}): JSX.Element { + const preset = providerPreset(provider) + const [values, setValues] = useState(() => initialValues(provider, existing)) + const idBase = useId() + const kept = existing?.secrets ?? {} + const set = (k: string, v: string | boolean): void => { + const next = { ...values, [k]: v } + setValues(next) + const p = profileFromValues(provider, next, kept) + onChange(p, profileProblem(p)) + } + + return ( +
+ {preset.kind === 'ipfs-kubo' || preset.kind === 'ipfs-pinning-service' ? ( +
+ + + kubo’s RPC API is the node’s admin interface. Letting this web app call it means trusting this site with your node, unless kubo limits the token it uses. Create a token restricted to add, pin-check, unpin and id (kubo API.Authorizations with AllowedPaths; the test prints the commands if the node refuses this origin) and paste it as the API Authorization header. + +
+ ) : null} + {fieldsFor(provider).map((f) => { + const hint = preset.hints[f.key] + const id = `${idBase}-${f.key}` + const keepable = f.secret === true && existing !== null && kept[f.key as keyof ProfileSecrets] !== undefined + return ( +
+ + set(f.key, e.target.value)} + aria-describedby={hint || f.secret ? `${id}-hint` : undefined} + /> + {hint || f.secret ? ( +

+ {f.secret ? SECRET_NOTE : null} + {keepable ? ` ${KEEP_NOTE}` : null} + {f.secret && hint ? ' ' : null} + {hint ? ( + <> + {hint.text} + {hint.href ? ( + <> + {' '} + + where to find this + + + ) : null} + + ) : null} +

+ ) : null} +
+ ) + })} + {preset.kind === 's3' ? ( + + ) : null} +
+ ) +} diff --git a/forge-web/components/storage/repo-storage-policy.tsx b/forge-web/components/storage/repo-storage-policy.tsx new file mode 100644 index 000000000..f172392fe --- /dev/null +++ b/forge-web/components/storage/repo-storage-policy.tsx @@ -0,0 +1,127 @@ +'use client' + +/** + * Repo Settings → Storage for browser pushes: which of the viewer's own storage profiles this + * repo's browser pushes (merges, fork updates, release assets) use, overriding the default + * from `/settings/storage`. Local to this browser: the credentials are, so the choice is too + * (the CLI keeps its equivalent in git config, `dash.storage`). + */ + +import { useState } from 'react' +import Link from 'next/link' +import { HardDrive } from 'lucide-react' +import { choiceOf, policyFor, policyForRepo, policyProblem, withRepoPolicy, type ReplicationChoice } from '@/lib/storage' +import { errText } from '@/lib/storage/util' +import { useStorageConfig } from '@/hooks/use-storage-config' +import { Button } from '@/components/ui/button' + +export function RepoStoragePolicy({ repoId }: { repoId: string }): JSX.Element { + const { config, storable, save, error } = useStorageConfig() + const override = config?.repoPolicies[repoId] ?? null + const effective = config ? policyForRepo(config, repoId) : null + const [targets, setTargets] = useState(null) + const [choice, setChoice] = useState(null) + const [msg, setMsg] = useState(null) + + if (error) return

{error}

+ if (!config) { + return ( +

+ Sign in to choose where your browser pushes to this repo go. Settings stay in this browser. +

+ ) + } + if (config.profiles.length === 0) { + return ( +

+ No storage set up in this browser: browser pushes here would store packs on Platform at ~0.28 DASH/MiB, asking first.{' '} + Set up storage → +

+ ) + } + const shown = targets ?? effective?.targets.slice() ?? [] + const shownChoice = choice ?? (effective ? choiceOf(effective) : 'one') + const policy = shown.length > 0 ? policyFor(shown, shownChoice) : null + const problem = policy ? policyProblem(config, policy) : null + + return ( +
+

+ + + Where your browser pushes to this repo store packs.{' '} + {override ? 'This repo has its own choice.' : effective ? 'Using your default.' : 'No default is set.'} Only you see this: it lives in this browser with your storage keys. + +

+
+ Storage for this repo + {config.profiles.map((p) => ( + + ))} +
+ +
+ + {override ? ( + + ) : null} + {msg ?? (problem && shown.length > 0 ? problem : '')} +
+
+ ) +} diff --git a/forge-web/components/storage/storage-test.tsx b/forge-web/components/storage/storage-test.tsx new file mode 100644 index 000000000..ff09171d8 --- /dev/null +++ b/forge-web/components/storage/storage-test.tsx @@ -0,0 +1,136 @@ +'use client' + +/** + * The live test rows of the storage wizard (`ux-dx-spec.md` §3.1 step 3). Each row settles to + * ok / FAIL / skipped with a plain reason; a CORS failure expands into the copy-paste fix for + * the provider, prefilled with the bucket and this app's origin, with a Re-test button. + */ + +import { useEffect, useRef, useState } from 'react' +import { sha256 } from '@noble/hashes/sha2.js' +import { bytesToHex } from '@noble/hashes/utils.js' +import { AlertTriangle, CheckCircle2, CircleDashed, Loader2, MinusCircle, RotateCw, XCircle } from 'lucide-react' +import { + IPFS_ROWS, + S3_ROWS, + corsFix, + probeProfile, + type ProbeRow, + type RowId, + type RowState, + type StorageProfile, +} from '@/lib/storage' +import { errText } from '@/lib/storage/util' +import { Button } from '@/components/ui/button' +import { CopyBlock } from '@/components/storage/copy-block' +import { cn } from '@/lib/utils' + +const META: Readonly> = { + pending: { word: 'waiting', klass: 'text-anvil-500 dark:text-anvil-400', Icon: CircleDashed }, + running: { word: 'testing', klass: 'text-anvil-500 dark:text-anvil-400', Icon: Loader2 }, + ok: { word: 'ok', klass: 'text-verify-700 dark:text-verify', Icon: CheckCircle2 }, + fail: { word: 'FAIL', klass: 'text-danger-700 dark:text-danger-400', Icon: XCircle }, + skipped: { word: 'skipped', klass: 'text-anvil-500 dark:text-anvil-400', Icon: MinusCircle }, +} + +function rowsFor(p: StorageProfile): ProbeRow[] { + const spec = p.settings.kind === 's3' ? S3_ROWS : IPFS_ROWS + return spec.map((r) => ({ ...r, state: 'pending', detail: '' })) +} + +/** A stable fingerprint of what a test ran against (settings and secrets). */ +export function profileSnapshot(p: StorageProfile): string { + // A digest, not the values: it becomes a React key and sits in state, where dev tools show it. + return bytesToHex(sha256(new TextEncoder().encode(JSON.stringify([p.name, p.settings, p.secrets])))) +} + +/** + * Run and show the live test for `profile`. `onDone(ok, snapshot)` reports whether every row + * passed, with the {@link profileSnapshot} the run tested, so the caller credits the result only + * to those exact values. The parent keys this component by the snapshot, so an edit remounts it + * (no stale green rows); a run still in flight when that happens reports into the unmounted + * copy and is dropped. + */ +export function StorageTest({ profile, onDone }: { profile: StorageProfile; onDone: (ok: boolean, snapshot: string) => void }): JSX.Element { + const [rows, setRows] = useState(null) + const [running, setRunning] = useState(false) + const [summary, setSummary] = useState('') + const mounted = useRef(true) + useEffect(() => { + mounted.current = true + return () => { + mounted.current = false + } + }, []) + const origin = typeof window === 'undefined' ? 'https://forge.dashhq.org' : window.location.origin + + const run = async (): Promise => { + if (running) return + const snapshot = profileSnapshot(profile) + setRunning(true) + setSummary('Testing…') + setRows(rowsFor(profile)) + const report = (id: RowId, state: RowState, detail: string, cors?: boolean): void => { + if (!mounted.current) return + setRows((prev) => (prev ?? []).map((r) => (r.id === id ? { ...r, state, detail, ...(cors ? { cors } : {}) } : r))) + } + let ok = false + try { + ok = await probeProfile(profile, report) + } catch (e) { + if (mounted.current) setRows((prev) => (prev ?? []).map((r) => (r.state === 'running' || r.state === 'pending' ? { ...r, state: 'fail', detail: errText(e) } : r))) + } + if (!mounted.current) return + setRunning(false) + setSummary(ok ? 'Test finished: every check passed.' : 'Test finished: some checks failed.') + onDone(ok, snapshot) + } + + const corsFailed = rows?.some((r) => r.state === 'fail' && r.cors) ?? false + const fix = corsFix(profile.settings.provider, profile.settings.kind === 's3' ? profile.settings.bucket : '', origin) + + return ( +
+
+ + + Runs from this page, so it checks what the browser will do: a few bytes are written and removed again. + +
+ {rows ? ( +
    + {rows.map((r) => { + const m = META[r.state] + return ( +
  1. + + {r.label} + {m.word} + {r.detail} +
  2. + ) + })} +
+ ) : null} +

+ {summary} +

+ {corsFailed && fix.text ? ( +
+

+ + + Fix CORS. {fix.where} + +

+ +

+ Reads are allowed from any origin (the objects are public); writes only from {origin}. Then press Re-test. +

+
+ ) : null} +
+ ) +} diff --git a/forge-web/components/storage/storage-wizard.tsx b/forge-web/components/storage/storage-wizard.tsx new file mode 100644 index 000000000..e1dfbd03c --- /dev/null +++ b/forge-web/components/storage/storage-wizard.tsx @@ -0,0 +1,430 @@ +'use client' + +/** + * `/settings/storage` — the storage wizard (`ux-dx-spec.md` §3.1): provider tiles, the + * per-provider form, the live browser test, then save; the saved profiles; where browser + * pushes go by default (replication radio); and the cost card, always visible. + * + * Everything saved is sealed in the vault with this browser's key (`lib/storage/store.ts`). + */ + +import { useId, useState } from 'react' +import { Cloud, Database, HardDrive, Link2, Pencil, Server, Trash2, Waypoints } from 'lucide-react' +import { + PROVIDERS, + choiceOf, + policyFor, + policyProblem, + providerPreset, + PLATFORM_PROFILE, + profileProblem, + withProfile, + withRenamedProfile, + withoutProfile, + type ProviderId, + type ReplicationChoice, + type StorageConfig, + type StorageProfile, +} from '@/lib/storage' +import { errText } from '@/lib/storage/util' +import { useStorageConfig } from '@/hooks/use-storage-config' +import { Button } from '@/components/ui/button' +import { ErrorState, LoadingBlock } from '@/components/ui/states' +import { ProfileForm, initialDraft } from '@/components/storage/profile-form' +import { StorageTest, profileSnapshot } from '@/components/storage/storage-test' +import { CostCard } from '@/components/storage/cost-card' +import { cn } from '@/lib/utils' +import { formatDate } from '@/lib/view/format' + +const TILE_ICON: Readonly> = { + r2: Cloud, + b2: Database, + aws: Server, + minio: HardDrive, + kubo: Waypoints, + pinning: Link2, + platform: Database, +} + +export function StorageWizard(): JSX.Element { + const { config, loading, error, storable, save, reload, discard } = useStorageConfig() + const [provider, setProvider] = useState(null) + const [editing, setEditing] = useState(null) + + if (loading && !config) return + if (error) return + if (!config) return + + const start = (p: ProviderId, existing: StorageProfile | null = null): void => { + setEditing(existing) + setProvider(p) + } + + return ( +
+ {!storable ? ( +

+ This tab signs with a pasted key, which has no vault. Storage settings are kept encrypted with a stored key: sign in by importing your identity to save them. +

+ ) : null} + +
+

+ Add storage +

+
+ {PROVIDERS.map((p) => { + const Icon = TILE_ICON[p.id] + const active = provider === p.id + return ( + + ) + })} +
+
+ + {provider ? ( + { + setProvider(null) + setEditing(null) + }} + /> + ) : null} + + start(p.settings.provider, p)} /> + + + + +
+ ) +} + +/** + * The stored settings could not be opened (sealed under an earlier key, or unreadable). Offer a + * retry and, since nothing else can open them, a way to discard them and start again. + */ +function UnreadableSettings({ message, onRetry, onDiscard }: { message: string; onRetry: () => void; onDiscard: () => Promise }): JSX.Element { + const [err, setErr] = useState(null) + return ( +
+ +
+ If they were sealed with an earlier key, they cannot be opened any more. + +
+ {err ?

{err}

: null} +
+ ) +} + +function AddProfile({ + provider, + existing, + config, + storable, + save, + onDone, +}: { + provider: ProviderId + existing: StorageProfile | null + config: StorageConfig + storable: boolean + save: (c: StorageConfig) => Promise + onDone: () => void +}): JSX.Element { + const preset = providerPreset(provider) + // An edit starts from the saved profile, so it can be tested before any field changes. + const [draft, setDraft] = useState<{ profile: StorageProfile; problem: string | null } | null>(() => { + if (existing === null || provider === 'platform') return null + const profile = initialDraft(provider, existing) + return { profile, problem: profileProblem(profile) } + }) + // A test result counts only for the exact values it ran against. + const [tested, setTested] = useState<{ ok: boolean; snapshot: string } | null>(null) + const [saving, setSaving] = useState(false) + const [saveError, setSaveError] = useState(null) + const statusId = useId() + + if (provider === 'platform') { + const has = config.profiles.some((p) => p.settings.kind === 'platform') + return ( +
+

Dash Platform

+

+ Packs stored as Platform documents: permanent, readable by anyone, nothing to run. Priced at about 0.28 DASH per MiB, charged when you push, and never refunded (Platform storage cannot be deleted, so nobody can break a repo others depend on). Every Platform write asks first, with its price. +

+
+ + +
+ {saveError ?

{saveError}

: null} +
+ ) + } + + const nameTaken = draft !== null && existing?.name !== draft.profile.name && config.profiles.some((p) => p.name === draft.profile.name) + const problem = draft?.problem ?? (draft ? null : 'fill in the fields') + const blocking = problem ?? (nameTaken ? 'a profile with that name exists' : null) + const snapshot = draft ? profileSnapshot(draft.profile) : '' + const result = tested !== null && tested.snapshot === snapshot ? tested.ok : null + const status = + blocking ?? (!storable ? 'Sign in with a stored key to save.' : result === null ? 'Run the test first.' : result ? 'All checks passed.' : 'Some checks failed: you can still save, but browser pushes need them all.') + + return ( +
+
+

{existing ? `Edit ${existing.name}` : preset.title}

+ {preset.blurb} +
+ setDraft({ profile, problem: p })} + /> + {draft && !blocking ? ( + setTested({ ok, snapshot: snap })} /> + ) : null} +
+ + + + {status} + +
+ {saveError ?

{saveError}

: null} +
+ ) +} + +function Profiles({ config, storable, save, onEdit }: { config: StorageConfig; storable: boolean; save: (c: StorageConfig) => Promise; onEdit: (p: StorageProfile) => void }): JSX.Element { + const [error, setError] = useState(null) + const remove = async (name: string): Promise => { + setError(null) + try { + await save(withoutProfile(config, name)) + } catch (e) { + setError(`Could not remove ${name}: ${errText(e)}`) + } + } + return ( +
+ {error ? ( +

+ {error} +

+ ) : null} +

+ Your storage +

+ {config.profiles.length === 0 ? ( +

+ No storage yet. Browser pushes (merges, forks) would store packs on Platform at ~0.28 DASH/MiB, asking first. +

+ ) : ( +
    + {config.profiles.map((p) => { + const test = config.lastTests[p.name] + const where = p.settings.kind === 's3' ? `${p.settings.bucket} · ${new URL(p.settings.endpoint).host}` : p.settings.kind === 'platform' ? 'on-chain chunks' : new URL(p.settings.api).host + return ( +
  • + {p.name} + {where} + {test ? ( + + {test.ok ? 'passed' : 'failed some checks'} · {formatDate(test.at)} + + ) : p.settings.kind === 'platform' ? null : ( + not tested + )} + + {p.settings.kind !== 'platform' ? ( + + ) : null} + + +
  • + ) + })} +
+ )} +
+ ) +} + +const CHOICES: readonly { id: ReplicationChoice; label: string }[] = [ + { id: 'one', label: 'One place: the first that confirms is enough' }, + { id: 'all', label: 'Every chosen place: a push fails unless all confirm' }, + { id: 'fallback', label: 'Platform as fallback if my storage fails (costed, asks first)' }, +] + +function DefaultPolicy({ config, storable, save }: { config: StorageConfig; storable: boolean; save: (c: StorageConfig) => Promise }): JSX.Element | null { + const current = config.defaultPolicy + const [targets, setTargets] = useState(() => current?.targets.slice() ?? []) + const [choice, setChoice] = useState(() => (current ? choiceOf(current) : 'one')) + const [saved, setSaved] = useState(false) + const [err, setErr] = useState(null) + if (config.profiles.length === 0) return null + const policy = targets.length > 0 ? policyFor(targets, choice) : null + const problem = policy ? policyProblem(config, policy) : null + + return ( +
+

+ Where browser pushes go +

+

+ The default for every repo you push to from this browser (merges, fork updates, release assets). A repo can override it under its Settings → Storage. The CLI keeps its own policy in git config. +

+
+ Storage to push to + {config.profiles.map((p) => ( + + ))} +
+
+ Replication + {CHOICES.map((c) => ( + + ))} +
+
+ + + {saved ? 'Saved.' : problem && targets.length > 0 ? problem : ''} + + {err ? {err} : null} +
+
+ ) +} diff --git a/forge-web/components/ui/button.tsx b/forge-web/components/ui/button.tsx index 0b4314a44..718b6558f 100644 --- a/forge-web/components/ui/button.tsx +++ b/forge-web/components/ui/button.tsx @@ -25,7 +25,7 @@ const button = cva( ghost: 'bg-transparent text-anvil-700 hover:bg-anvil-100 dark:text-anvil-200 dark:hover:bg-anvil-800', danger: - 'border border-danger/40 bg-transparent text-danger hover:bg-danger/10', + 'border border-danger/40 bg-transparent text-danger-700 hover:bg-danger/10 dark:text-danger-400', subtle: 'bg-anvil-100 text-anvil-800 hover:bg-anvil-200 dark:bg-anvil-800 dark:text-anvil-100 dark:hover:bg-anvil-750', }, diff --git a/forge-web/contexts/auth-context.tsx b/forge-web/contexts/auth-context.tsx index 961254fd1..23dd1c685 100644 --- a/forge-web/contexts/auth-context.tsx +++ b/forge-web/contexts/auth-context.tsx @@ -105,6 +105,14 @@ export function AuthProvider({ useEffect(() => controller.subscribe(setState), [controller]) + // A notice from a sign-in step (e.g. storage settings that could not survive a key renewal). + const notice = state.notice ?? null + useEffect(() => { + if (!notice) return + toast({ title: 'Storage settings were not carried over', tone: 'warn', detail: notice }) + controller.clearNotice() + }, [notice, controller]) + const [vaults, setVaults] = useState([]) const reloadVaults = useCallback(() => { controller.storedVaults().then(setVaults, () => setVaults([])) diff --git a/forge-web/e2e/helpers.ts b/forge-web/e2e/helpers.ts index 0176d324a..180980aea 100644 --- a/forge-web/e2e/helpers.ts +++ b/forge-web/e2e/helpers.ts @@ -152,7 +152,7 @@ export const PASSPHRASE = 'e2e passphrase for the vault' * per identity instead of registering a new one on every test (identities would otherwise * accumulate keys without bound). Gitignored (e2e/.playwright/). */ -function stateFile(name: string): string { +export function stateFile(name: string): string { return join(__dirname, '.playwright', 'auth', `devnet-${E2E_DEVNET}-${name}.json`) } diff --git a/forge-web/e2e/storage-wizard.spec.ts b/forge-web/e2e/storage-wizard.spec.ts new file mode 100644 index 000000000..dd24f6d51 --- /dev/null +++ b/forge-web/e2e/storage-wizard.spec.ts @@ -0,0 +1,139 @@ +import { test, expect, type Page } from '@playwright/test' +import { existsSync } from 'node:fs' +import { E2E_DEVNET, idFile, runAxe, shot, signedIn, stateFile, unlock } from './helpers' + +/** + * The storage wizard (`/settings/storage`) against the local MinIO of infra/docker-compose.yml + * (`forge-byo`: SigV4-signed writes, anonymous reads, CORS for any origin), live on a devnet + * build because a vault needs a signed-in identity: + * + * docker compose -f infra/docker-compose.yml up -d minio minio-init static-http + * E2E_DEVNET=moutai E2E_WRITE=1 pnpm exec playwright test storage-wizard.spec.ts + * + * s1. on MinIO every write, read, range and CORS row passes, and the public-read row fails with + * the reason a loopback address cannot be published; the profile is saved into the vault, + * never into localStorage; it survives a reload + unlock; a default policy is set. + * s2. a reachable host with no CORS (the static nginx) fails the CORS rows and shows the + * copy-paste fix, prefilled with the bucket and this origin. + * s3. a repo's own policy (its Settings → Your browser pushes). + * s4. axe and a 390 px viewport. + * + * Signs in as MAINTAINER (first run: registers one limited key; later runs reuse the vault). + */ + +test.skip(E2E_DEVNET === '' || process.env['E2E_WRITE'] !== '1', 'needs a devnet build and a signed-in vault: E2E_DEVNET=moutai E2E_WRITE=1') +test.skip(!existsSync(idFile('MAINTAINER')), 'devnet test identities not found') +test.describe.configure({ mode: 'serial', timeout: 240_000 }) + +const MINIO = { endpoint: 'http://127.0.0.1:9000', bucket: 'forge-byo', publicUrl: 'http://127.0.0.1:9000/forge-byo', key: 'minioadmin', secret: 'minioadmin' } +const EMPTY_REPO = { owner: 'GKBTXUdo3MpRYAUqgZvTZGTav9mXGqfJfR5822K2tp79', name: 'forge-v2-empty' } + +async function fillS3(page: Page, v: { name: string; endpoint: string; bucket: string; publicUrl: string }): Promise { + await page.getByLabel('Profile name', { exact: true }).fill(v.name) + await page.getByLabel('S3 endpoint', { exact: true }).fill(v.endpoint) + await page.getByLabel('Region', { exact: true }).fill('us-east-1') + await page.getByLabel('Bucket', { exact: true }).fill(v.bucket) + await page.getByLabel('Public URL', { exact: true }).fill(v.publicUrl) + await page.getByLabel(/^Key prefix/).fill('e2e-web') + await page.getByLabel('Access key id', { exact: true }).fill(MINIO.key) + await page.getByLabel('Secret access key', { exact: true }).fill(MINIO.secret) +} + +test('s1. MinIO passes every browser check, and the profile is sealed in the vault', async ({ browser }) => { + const page = await signedIn(browser, 'MAINTAINER', '/settings/storage/') + // A rerun: remove what an earlier run saved, so this one starts from an empty list. + await expect(page.getByRole('heading', { name: 'Your storage' })).toBeVisible({ timeout: 30_000 }) + page.on('dialog', (d) => void d.accept()) + while ((await page.getByRole('button', { name: /^remove /i }).count()) > 0) { + await page.getByRole('button', { name: /^remove /i }).first().click() + await page.waitForTimeout(500) + } + await shot(page, 'a-01-storage-empty') + await page.getByTestId('tile-minio').click() + await fillS3(page, { name: 'minio-e2e', ...MINIO }) + await page.getByRole('button', { name: /^test$/i }).click() + for (const row of ['put', 'get', 'range', 'cors-put', 'delete']) { + await expect(page.getByTestId(`probe-${row}`)).toHaveAttribute('data-state', 'ok', { timeout: 60_000 }) + } + // The anonymous read works from here, but 127.0.0.1 is not an address anyone else can read: + // it would be recorded on chain, so the row fails and says why (uploads refuse it too). + const publicRow = page.getByTestId('probe-public') + await expect(publicRow).toHaveAttribute('data-state', 'fail') + await expect(publicRow).toContainText(/Readable from here, but 127\.0\.0\.1:9000 is only reachable from this machine/) + await expect(page.getByTestId('cors-fix')).toHaveCount(0) + await shot(page, 'a-02-minio-tested') + await page.getByRole('button', { name: /save profile/i }).click() + await expect(page.getByTestId('profile-list')).toContainText('minio-e2e') + await expect(page.getByTestId('profile-list')).toContainText('failed some checks') + + // Default policy: this profile, one copy. + await page.getByRole('checkbox', { name: 'minio-e2e' }).first().check() + await page.getByRole('radio', { name: /one place/i }).check() + await page.getByRole('button', { name: /save default/i }).click() + await expect(page.getByText('Saved.')).toBeVisible() + // Later tests restore this context's IndexedDB: keep the (sealed) settings in it. + await page.context().storageState({ path: stateFile('MAINTAINER'), indexedDB: true }) + + // Secrets never reach localStorage; IndexedDB holds only ciphertext. + const stores = await page.evaluate(async () => { + const ls = JSON.stringify(Object.entries(localStorage)) + const idb = await new Promise((resolve) => { + const req = indexedDB.open('dash-forge') + req.onsuccess = () => { + const tx = req.result.transaction('vault', 'readonly') + const all = tx.objectStore('vault').getAll() + all.onsuccess = () => resolve(JSON.stringify(all.result, (_k, v: unknown) => (v instanceof Uint8Array ? `<${v.length} bytes>` : v))) + } + }) + return { ls, idb } + }) + for (const s of [stores.ls, stores.idb]) { + expect(s).not.toContain('minioadmin') + expect(s).not.toContain('forge-byo') + } + + // A reload locks the vault; the settings come back after unlocking. + await page.reload({ waitUntil: 'domcontentloaded' }) + await unlock(page) + await expect(page.getByTestId('profile-list')).toContainText('minio-e2e', { timeout: 30_000 }) + await shot(page, 'a-03-profile-saved') +}) + +test('s2. a reachable host without CORS shows the copy-paste fix', async ({ browser }) => { + const page = await signedIn(browser, 'MAINTAINER', '/settings/storage/') + await page.getByTestId('tile-r2').click() + await fillS3(page, { name: 'no-cors', endpoint: 'http://127.0.0.1:8082', bucket: 'my-bucket', publicUrl: 'http://127.0.0.1:8082/my-bucket' }) + await page.getByRole('button', { name: /^test$/i }).click() + await expect(page.getByTestId('probe-cors-put')).toHaveAttribute('data-state', 'fail', { timeout: 60_000 }) + const fix = page.getByTestId('cors-fix') + await expect(fix).toBeVisible() + await expect(fix).toContainText('R2 → my-bucket → Settings → CORS Policy') + await expect(fix).toContainText('"AllowedMethods"') + await expect(fix).toContainText(new URL(page.url()).origin) + await expect(fix).toContainText('x-amz-content-sha256') + await shot(page, 'a-04-cors-fix') + await page.getByRole('button', { name: /cancel/i }).click() +}) + +test('s3. a repo can override where browser pushes go', async ({ browser }) => { + const page = await signedIn(browser, 'MAINTAINER', `/repo/settings/?owner=${EMPTY_REPO.owner}&name=${EMPTY_REPO.name}`) + const box = page.getByTestId('repo-storage-policy') + await expect(box).toContainText('Using your default', { timeout: 60_000 }) + await box.getByRole('combobox').selectOption('fallback') + await box.getByRole('button', { name: /use for this repo/i }).click() + await expect(box).toContainText('Saved for this repo.') + await expect(box).toContainText('This repo has its own choice.') + await shot(page, 'a-05-repo-policy') + await box.getByRole('button', { name: /use my default/i }).click() + await expect(box).toContainText('Back to your default.') +}) + +test('s4. accessible, and usable at 390 px', async ({ browser }) => { + const page = await signedIn(browser, 'MAINTAINER', '/settings/storage/') + await page.getByTestId('tile-r2').click() + expect(await runAxe(page, 'storage wizard')).toEqual([]) + await page.setViewportSize({ width: 390, height: 844 }) + await expect(page.getByText(/use a desktop browser for this step/i)).toBeVisible() + await expect(page.getByTestId('tile-minio')).toBeVisible() + await shot(page, 'a-06-mobile') +}) diff --git a/forge-web/hooks/use-storage-config.ts b/forge-web/hooks/use-storage-config.ts new file mode 100644 index 000000000..881094c38 --- /dev/null +++ b/forge-web/hooks/use-storage-config.ts @@ -0,0 +1,52 @@ +'use client' + +/** + * useStorageConfig — the signed-in identity's storage configuration, read from and written to + * the vault. Null while signed out or locked (the settings are sealed with the key; there is + * nothing to show without it). A tab-only pasted key has no vault record, so it cannot store + * settings; `storable` says so. + */ + +import { useCallback } from 'react' + +import { useAuth } from '@/contexts/auth-context' +import { useAsync } from '@/hooks/use-async' +import { DEFAULT_NETWORK } from '@/lib/constants' +import { EMPTY_STORAGE_CONFIG, discardStorageConfig, loadStorageConfig, saveStorageConfig, type StorageConfig } from '@/lib/storage' + +export interface StorageConfigState { + readonly config: StorageConfig | null + readonly loading: boolean + readonly error: string | null + /** Whether this session can store settings (a vault-stored key). */ + readonly storable: boolean + save: (next: StorageConfig) => Promise + reload: () => void + /** Delete stored settings this key cannot open, then reload (empty). */ + discard: () => Promise +} + +export function useStorageConfig(): StorageConfigState { + const { identity, storage } = useAuth() + const storable = identity !== null && storage === 'vault' + const state = useAsync( + () => (storable ? loadStorageConfig(DEFAULT_NETWORK, identity) : Promise.resolve(EMPTY_STORAGE_CONFIG)), + [identity ?? '', storage ?? ''], + { enabled: identity !== null }, + ) + const { reload } = state + const save = useCallback( + async (next: StorageConfig): Promise => { + if (!storable || identity === null) throw new Error('Storage settings are kept in the vault: sign in with a stored key (not a pasted one).') + await saveStorageConfig(DEFAULT_NETWORK, identity, next) + reload() + }, + [identity, reload, storable], + ) + const discard = useCallback(async (): Promise => { + if (identity === null) return + await discardStorageConfig(DEFAULT_NETWORK, identity) + reload() + }, [identity, reload]) + return { config: identity === null ? null : state.data, loading: state.loading, error: state.error, storable, save, reload, discard } +} diff --git a/forge-web/lib/auth/controller.ts b/forge-web/lib/auth/controller.ts index 25f479091..58773a529 100644 --- a/forge-web/lib/auth/controller.ts +++ b/forge-web/lib/auth/controller.ts @@ -72,6 +72,8 @@ export interface AuthState { readonly session: AuthSession | null readonly isLoading: boolean readonly error: string | null + /** Something the user should know after a sign-in step succeeded (null when nothing). */ + readonly notice?: string | null } type Listener = (state: AuthState) => void @@ -120,6 +122,11 @@ export class AuthController { return this.state } + /** The UI showed the notice: clear it, so the next one (even the same text) shows too. */ + clearNotice(): void { + if (this.state.notice) this.setState({ notice: null }) + } + subscribe(listener: Listener): () => void { this.listeners.add(listener) return () => { @@ -230,7 +237,12 @@ export class AuthController { */ private async adopt(identityId: string, key: LimitedKey, protection: Protection): Promise { const secret: VaultSecret = { identityId, keyId: key.keyId, wif: key.wif } - await storeInVault(this.network, secret, protection) + const { storageSettingsDropped } = await storeInVault(this.network, secret, protection) + if (storageSettingsDropped) { + this.setState({ + notice: 'Your storage settings were sealed with the previous key, which was locked when you renewed it, so they could not be carried over. Add your storage again in Settings → Storage.', + }) + } try { return await this.open(secret, 'vault', key.limits) } catch (e) { diff --git a/forge-web/lib/auth/vault.ts b/forge-web/lib/auth/vault.ts index 64c1fc018..32f032b0d 100644 --- a/forge-web/lib/auth/vault.ts +++ b/forge-web/lib/auth/vault.ts @@ -14,6 +14,11 @@ * record cannot be replayed under another identity. * - In memory: the unlocked record lives only in this module (never React state, never * localStorage), for at most {@link AUTO_LOCK_MS} (12 h) or until {@link lockVault}. + * - Storage credentials (`ux-dx-spec.md` §3.1: bucket keys, pinning tokens) are a second + * AES-GCM blob beside the record (`vault-storage::`), sealed under a key + * HKDF-derived from the same data key. While unlocked, only that derived key is held, as a + * non-extractable `CryptoKey`; it is dropped with the rest on lock. A renewal that replaces + * the record re-seals the blob under the new data key. * * Per-origin by construction: IndexedDB and WebAuthn (`rpId` = this host) are origin-scoped. */ @@ -23,7 +28,7 @@ import { hkdf } from '@noble/hashes/hkdf.js' import { sha256 } from '@noble/hashes/sha2.js' import type { Network } from '../constants' -import { idbDelete, idbEntries, idbGet, idbPut } from '../idb' +import { idbBatch, idbDelete, idbEntries, idbGet, idbPut } from '../idb' /** Unlocked vaults lock themselves after this long (spec §2.3). */ export const AUTO_LOCK_MS = 12 * 60 * 60 * 1000 @@ -112,6 +117,53 @@ function aad(network: Network, identityId: string, slot: string): Uint8Array { return enc.encode(`dash-forge vault v1|${network}|${identityId}|${slot}`) } +function storageBlobKey(network: Network, identityId: string): string { + return `vault-storage:${network}:${identityId}` +} + +/** The sealed storage-credentials blob. */ +interface StorageBlob { + readonly iv: Uint8Array + readonly ciphertext: Uint8Array +} + +/** The storage-blob key: HKDF(data key) → a non-extractable AES-GCM key. */ +async function deriveStorageKey(dataKey: Uint8Array, network: Network, identityId: string): Promise { + const raw = hkdf(sha256, dataKey, enc.encode('dash-forge vault storage v1'), enc.encode(`${network}|${identityId}`), 32) + try { + return await aesKey(raw, ['encrypt', 'decrypt']) + } finally { + raw.fill(0) + } +} + +async function sealBlob(key: CryptoKey, network: Network, identityId: string, value: unknown): Promise { + const iv = random(12) + const plain = enc.encode(JSON.stringify(value)) + try { + const ct = await crypto.subtle.encrypt({ name: 'AES-GCM', iv: buf(iv), additionalData: buf(aad(network, identityId, 'storage')) }, key, buf(plain)) + return { iv, ciphertext: new Uint8Array(ct) } + } finally { + plain.fill(0) + } +} + +async function openBlob(key: CryptoKey, network: Network, identityId: string, blob: StorageBlob): Promise { + let plain: Uint8Array + try { + plain = new Uint8Array( + await crypto.subtle.decrypt({ name: 'AES-GCM', iv: buf(blob.iv), additionalData: buf(aad(network, identityId, 'storage')) }, key, buf(blob.ciphertext)), + ) + } catch { + throw new VaultLockedError('the stored storage settings do not open with this key') + } + try { + return JSON.parse(new TextDecoder().decode(plain)) as unknown + } finally { + plain.fill(0) + } +} + function random(n: number): Uint8Array { return crypto.getRandomValues(new Uint8Array(n)) } @@ -237,18 +289,34 @@ export interface Protection { readonly passkey?: { credentialId: Uint8Array; prfSalt: Uint8Array; output: Uint8Array } } +/** What {@link storeInVault} did with storage settings sealed under the record it replaced. */ +export interface StoreOutcome { + /** + * Storage settings existed but could not be carried across (the vault was locked when the + * key was renewed, so they could not be opened). They are deleted; the user must add them + * again, and the UI says so. + */ + readonly storageSettingsDropped: boolean +} + /** * Seal `secret` into the vault for `network`, replacing any earlier record for its identity, - * and keep it unlocked for this session. At least one protection is required. + * and keep it unlocked for this session. At least one protection is required. The record and + * the (re-sealed or deleted) storage settings are written in one transaction. */ -export async function storeInVault(network: Network, secret: VaultSecret, protection: Protection): Promise { +export async function storeInVault(network: Network, secret: VaultSecret, protection: Protection): Promise { assertDedicatedOrigin() if (!protection.passphrase && !protection.passkey) throw new Error('the vault needs a passphrase or a passkey') if (protection.passphrase !== undefined && protection.passphrase.length < MIN_PASSPHRASE) { throw new Error(`use a passphrase of at least ${MIN_PASSPHRASE} characters`) } const { identityId } = secret + // A renewal replaces the record (and its data key): carry the storage settings across when + // this session can open them; otherwise they cannot be opened any more and are dropped. + const hadBlob = (await idbGet('vault', storageBlobKey(network, identityId))) !== undefined + const carried = hadBlob ? await readStorageBlob(network, identityId).catch(() => null) : null const dataKey = random(32) + let storageKey: CryptoKey try { const body = await seal(dataKey, enc.encode(JSON.stringify(secret)), aad(network, identityId, 'body')) const slots: Slot[] = [] @@ -275,11 +343,53 @@ export async function storeInVault(network: Network, secret: VaultSecret, protec ciphertext: body.ct, slots, } - await idbPut('vault', key(network, identityId), record) + storageKey = await deriveStorageKey(dataKey, network, identityId) + const blob = carried !== null ? await sealBlob(storageKey, network, identityId, carried) : undefined + // One transaction: a crash between the two writes must not leave settings sealed under a + // data key no record holds any more. + await idbBatch('vault', [ + [key(network, identityId), record], + [storageBlobKey(network, identityId), blob], + ]) } finally { dataKey.fill(0) } - setUnlocked(network, secret) + setUnlocked(network, secret, storageKey) + return { storageSettingsDropped: hadBlob && carried === null } +} + +/** + * Delete storage settings that cannot be opened (sealed under an earlier key, or unreadable), + * so the user can start over instead of being stuck on an error. + */ +export async function discardStorageBlob(network: Network, identityId: string): Promise { + await idbDelete('vault', storageBlobKey(network, identityId)) +} + +/** + * The storage settings sealed for (network, identity), or null when none are stored. Needs the + * vault unlocked for that identity ({@link VaultLockedError} otherwise). The caller parses the + * value (it is the caller's schema). + */ +export async function readStorageBlob(network: Network, identityId: string): Promise { + const k = unlockedStorageKey(network, identityId) + const blob = await idbGet('vault', storageBlobKey(network, identityId)) + if (blob === undefined) return null + if (k === null) throw new VaultLockedError('unlock to read your storage settings') + return openBlob(k, network, identityId, blob) +} + +/** Seal `value` as the storage settings of (network, identity). Needs the vault unlocked. */ +export async function writeStorageBlob(network: Network, identityId: string, value: unknown): Promise { + assertDedicatedOrigin() + const k = unlockedStorageKey(network, identityId) + if (k === null) throw new VaultLockedError('unlock with a stored key to save storage settings') + await idbPut('vault', storageBlobKey(network, identityId), await sealBlob(k, network, identityId, value)) +} + +function unlockedStorageKey(network: Network, identityId: string): CryptoKey | null { + if (unlockedSecret(network, identityId) === null) return null + return unlocked?.storageKey ?? null } /** The vaults stored for `network` (no secrets). */ @@ -293,14 +403,14 @@ export async function listVaults(network: Network): Promise { })) } -async function unwrapWith(network: Network, record: VaultRecord, kek: Uint8Array, slot: Slot): Promise { +async function unwrapWith(network: Network, record: VaultRecord, kek: Uint8Array, slot: Slot): Promise<{ secret: VaultSecret; storageKey: CryptoKey }> { const dataKey = await open(kek, slot.iv, slot.wrapped, aad(network, record.identityId, slot.kind)) try { const body = await open(dataKey, record.iv, record.ciphertext, aad(network, record.identityId, 'body')) const secret = JSON.parse(new TextDecoder().decode(body)) as VaultSecret body.fill(0) if (secret.identityId !== record.identityId) throw new VaultLockedError('vault record does not match its identity') - return secret + return { secret, storageKey: await deriveStorageKey(dataKey, network, record.identityId) } } finally { dataKey.fill(0) kek.fill(0) @@ -316,8 +426,8 @@ export async function unlockWithPassphrase(network: Network, identityId: string, // The parameters are pinned, never read from the (unauthenticated) record: a tampered // record cannot make unlock allocate gigabytes. if (JSON.stringify(slot.params) !== JSON.stringify(ARGON2_PARAMS)) throw new VaultLockedError('this key was stored with unsupported settings') - const secret = await unwrapWith(network, record, await passphraseKey(passphrase, slot.salt), slot) - setUnlocked(network, secret) + const { secret, storageKey } = await unwrapWith(network, record, await passphraseKey(passphrase, slot.salt), slot) + setUnlocked(network, secret, storageKey) return secret } @@ -332,8 +442,8 @@ export async function unlockWithPasskey(network: Network, identityId: string): P const raw = new Uint8Array(output) const kek = prfKey(raw, network, identityId) raw.fill(0) - const secret = await unwrapWith(network, record, kek, slot) - setUnlocked(network, secret) + const { secret, storageKey } = await unwrapWith(network, record, kek, slot) + setUnlocked(network, secret, storageKey) return secret } @@ -341,6 +451,7 @@ export async function unlockWithPasskey(network: Network, identityId: string): P export async function forgetVault(network: Network, identityId: string): Promise { lockVault() await idbDelete('vault', key(network, identityId)) + await idbDelete('vault', storageBlobKey(network, identityId)) clearSignedWrites() } @@ -359,8 +470,8 @@ export function clearSignedWrites(): void { } } -// The unlocked secret: module memory only. -let unlocked: { network: Network; secret: VaultSecret; at: number } | null = null +// The unlocked secret (and the storage-blob key, for a vault-stored key): module memory only. +let unlocked: { network: Network; secret: VaultSecret; at: number; storageKey: CryptoKey | null } | null = null let lockTimer: ReturnType | null = null const lockListeners = new Set<() => void>() @@ -372,8 +483,8 @@ export function onVaultLock(listener: () => void): () => void { } } -function setUnlocked(network: Network, secret: VaultSecret): void { - unlocked = { network, secret, at: Date.now() } +function setUnlocked(network: Network, secret: VaultSecret, storageKey: CryptoKey | null = null): void { + unlocked = { network, secret, at: Date.now(), storageKey } if (lockTimer) clearTimeout(lockTimer) lockTimer = setTimeout(lockVault, AUTO_LOCK_MS) } diff --git a/forge-web/lib/design/contrast.test.ts b/forge-web/lib/design/contrast.test.ts index 79725721c..3d04149bd 100644 --- a/forge-web/lib/design/contrast.test.ts +++ b/forge-web/lib/design/contrast.test.ts @@ -102,6 +102,21 @@ describe('Verification state words meet WCAG AA', () => { }) }) +describe('danger text meets WCAG AA', () => { + const danger = colors['danger'] as Record + it.each(Object.entries(LIGHT_SURFACES))('danger-700 on %s (light theme)', (_, bg) => { + expect(contrast(rgb(danger['700']!), rgb(bg!))).toBeGreaterThanOrEqual(AA_TEXT) + }) + it.each(Object.entries(DARK_SURFACES))('danger-400 on %s (dark theme)', (_, bg) => { + expect(contrast(rgb(danger['400']!), rgb(bg!))).toBeGreaterThanOrEqual(AA_TEXT) + }) + it('the base value is not a text color on the darkest or lightest-gray surface', () => { + // 4.01:1 on anvil-950: the storage wizard's axe failure (a danger button). + expect(contrast(rgb(danger.DEFAULT!), rgb(anvil['950']!))).toBeLessThan(AA_TEXT) + expect(contrast(rgb(danger.DEFAULT!), rgb(anvil['100']!))).toBeLessThan(AA_TEXT) + }) +}) + describe('no component renders text in the raw brand blue', () => { const root = resolve(__dirname, '../..') diff --git a/forge-web/lib/idb.ts b/forge-web/lib/idb.ts index def2802ac..39caaac84 100644 --- a/forge-web/lib/idb.ts +++ b/forge-web/lib/idb.ts @@ -85,6 +85,33 @@ export async function idbPut(store: StoreName, key: string, value: T): Promis await run(store, 'readwrite', (s) => s.put(value, key)) } +/** + * Several puts and deletes in ONE store, atomically: one transaction, so a crash or a failed + * write leaves either all of them or none (`value: undefined` deletes the key). + */ +export async function idbBatch(store: StoreName, ops: readonly (readonly [string, unknown])[]): Promise { + if (!hasIndexedDb()) { + const m = mem(store) + for (const [k, v] of ops) { + if (v === undefined) m.delete(k) + else m.set(k, structuredClone(v)) + } + return + } + const db = await open() + await new Promise((resolve, reject) => { + const tx = db.transaction(store, 'readwrite') + const s = tx.objectStore(store) + for (const [k, v] of ops) { + if (v === undefined) s.delete(k) + else s.put(v, k) + } + tx.oncomplete = () => resolve() + tx.onerror = () => reject(tx.error ?? new Error('IndexedDB transaction failed')) + tx.onabort = () => reject(tx.error ?? new Error('IndexedDB transaction aborted')) + }) +} + /** Delete one value (no-op when absent). */ export async function idbDelete(store: StoreName, key: string): Promise { if (!hasIndexedDb()) { diff --git a/forge-web/lib/net.ts b/forge-web/lib/net.ts new file mode 100644 index 000000000..a301fc691 --- /dev/null +++ b/forge-web/lib/net.ts @@ -0,0 +1,52 @@ +/** + * Which URLs may be recorded on chain, and which a reader may fetch. + * + * Anything a `packManifest` records is read by every visitor's browser, so it must be a public + * https URL: never plain http, never this machine or a private network. Otherwise one owner's + * manifest could make every reader's browser send requests to its own local services (blind + * GET-CSRF, port probing), and a copy only its uploader can reach would count as "stored". + * Loopback http is fine only for what the user's own browser talks to directly while setting + * up storage (a local kubo API, a local MinIO endpoint), never for what gets published. + */ + +/** + * Whether `hostname` names this machine or a private / link-local network. + * + * Literal addresses and reserved names only: a page cannot resolve DNS, so a public name that + * resolves to a private address (`127.0.0.1.nip.io`, DNS rebinding) passes. That residual is + * bounded by the rest of the design: every byte read is hash-checked, requests carry no + * credentials, and responses are opaque to other origins. + */ +export function isPrivateHost(hostname: string): boolean { + const h = hostname.toLowerCase().replace(/^\[|\]$/g, '') + if (h === 'localhost' || h.endsWith('.localhost') || h.endsWith('.local') || h.endsWith('.internal')) return true + const v4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(h) + if (v4) { + const [a, b] = [Number(v4[1]), Number(v4[2])] + return a === 0 || a === 10 || a === 127 || (a === 100 && b >= 64 && b <= 127) || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) + } + if (h.includes(':')) { + // IPv6: loopback, unspecified, unique-local fc00::/7, link-local fe80::/10, v4-mapped. + return h === '::1' || h === '::' || /^f[cd]/.test(h) || /^fe[89ab]/.test(h) || h.startsWith('::ffff:') + } + return false +} + +/** Whether `url` is a public https URL (what readers may fetch and a manifest may record). */ +export function isPublicHttpsUrl(url: string): boolean { + try { + const u = new URL(url) + return u.protocol === 'https:' && !isPrivateHost(u.hostname) && u.username === '' && u.password === '' + } catch { + return false + } +} + +/** + * Whether a manifest may record `uri`: a public https URL, or a non-HTTP locator browsers do + * not fetch directly (`ipfs://`, `s3://`, `platform://`). + */ +export function isRecordableUri(uri: string): boolean { + if (/^(ipfs|s3|platform):\/\//i.test(uri)) return true + return isPublicHttpsUrl(uri) +} diff --git a/forge-web/lib/repo/push.ts b/forge-web/lib/repo/push.ts new file mode 100644 index 000000000..c408d963f --- /dev/null +++ b/forge-web/lib/repo/push.ts @@ -0,0 +1,211 @@ +/** + * Push-side chain writes for forge-v2 repos: `packManifest` and Platform `chunk` documents, + * shaped exactly as forge-core writes them (`RepoService::write_pack_manifest`, + * `backends/platform.rs::encode_chunk_doc`, `pack::split`), so a pack the browser records is + * read by the CLI and vice versa. + * + * Every write is member-gated at consensus (`ownerRefersTo` maintainer or writer) and + * idempotent here: the unique indexes are `(repoId, $ownerId, packHash)` for a manifest and + * `(repoId, $ownerId, packHash, seq)` for a chunk, so each checks the signer's own copy first + * and a duplicate-unique refusal (a lost answer to an earlier attempt) counts as done. + */ + +import type { EvoSDK } from '@dashevo/evo-sdk' +import { hexToBytes } from '@noble/hashes/utils.js' + +import { CHUNK_FIELDS, FIELD_MAX, MANIFEST_MAX_URIS, MANIFEST_URI_MAX_LEN } from '../constants' +import { decodeIdentifier } from '../auth/base58' +import { + ConsensusRefusal, + DUPLICATE_UNIQUE_CODE, + createDocumentIdempotent, + hexToBase64, + previewCredits, + queryDocumentsWithProof, + type WriteAuth, + type WriteResult, +} from '../sdk' +import { DOC, type V2RepoRef } from './contract' +import { repoSource } from './source' + +/** The fields of a `packManifest` (forge-core `PackManifestInput`). */ +export interface PackManifestInput { + /** Hex SHA-256 of the artifact. */ + readonly packHash: string + /** 0 git pack | 1 objectLocator | 2 flatIndex. */ + readonly kind: number + readonly sizeBytes: number + readonly objectCount: number + /** Platform chunk documents holding a copy (0 when none). */ + readonly chunkCount: number + /** 0 = chunks on Platform, 1 = external only. */ + readonly storage: 0 | 1 + /** Where the bytes are: a `platform://` locator first when chunks exist, then public URLs. */ + readonly uris: readonly string[] + /** flatIndex only: the tip commit oids it indexes (hex). */ + readonly tips?: readonly string[] + /** Hex pack hashes this one supersedes. */ + readonly supersedes?: readonly string[] +} + +/** Why `uris` does not fit the manifest's typed array, or null (`uris` ≤ 8 × ≤ 300 bytes). */ +export function manifestUrisProblem(uris: readonly string[]): string | null { + if (uris.length === 0) return 'no confirmed copy recorded any URI; refusing to write a manifest nothing can read' + if (uris.length > MANIFEST_MAX_URIS) return `a manifest holds at most ${MANIFEST_MAX_URIS} URIs` + if (uris.some((u) => new TextEncoder().encode(u).length > MANIFEST_URI_MAX_LEN)) return `a manifest URI holds at most ${MANIFEST_URI_MAX_LEN} bytes` + return null +} + +function concatHex(list: readonly string[], width: number): Uint8Array { + const out = new Uint8Array(list.length * width) + list.forEach((h, i) => { + const b = hexToBytes(h) + if (b.length !== width) throw new Error(`expected ${width}-byte entries, got ${b.length}`) + out.set(b, i * width) + }) + return out +} + +/** The result for a manifest this signer already recorded (nothing spent). */ +function alreadyRecorded(documentId: string): WriteResult { + return { documentId, confirmed: true, cost: previewCredits(0), actualCredits: 0 } +} + +function isDuplicate(e: unknown): boolean { + return e instanceof ConsensusRefusal && e.code === DUPLICATE_UNIQUE_CODE +} + +/** The signer's own manifest of `packHash` in `repo` (the unique slot), or null. */ +export async function findOwnManifest(sdk: EvoSDK, repo: V2RepoRef, ownerId: string, packHashHex: string): Promise { + const { documents } = await queryDocumentsWithProof( + sdk, + repoSource(repo).repoQuery(DOC.packManifest, { + where: [ + ['$ownerId', '==', ownerId], + ['packHash', '==', hexToBase64(packHashHex)], + ], + limit: 1, + }), + ) + const id = documents[0]?.['$id'] + return typeof id === 'string' ? id : null +} + +/** + * Write a `packManifest` (member-gated). Returns the signer's manifest id: an existing one + * (nothing spent) when this signer already recorded the pack. + */ +export async function writePackManifest( + sdk: EvoSDK, + auth: WriteAuth, + repo: V2RepoRef, + input: PackManifestInput, + intent?: string, +): Promise { + const problem = manifestUrisProblem(input.uris) + if (problem) throw new Error(problem) + const existing = await findOwnManifest(sdk, repo, auth.identityId, input.packHash) + if (existing !== null) return alreadyRecorded(existing) + const data: Record = { + repoId: decodeIdentifier(repo.repoId), + packHash: hexToBytes(input.packHash), + kind: input.kind, + sizeBytes: input.sizeBytes, + objectCount: input.objectCount, + chunkCount: input.chunkCount, + storage: input.storage, + offsetIndexParts: 0, + uris: [...input.uris], + } + if (input.tips && input.tips.length > 0) data['tips'] = concatHex(input.tips, 20) + if (input.supersedes && input.supersedes.length > 0) data['supersedes'] = concatHex(input.supersedes, 32) + try { + return await createDocumentIdempotent(sdk, auth, { contractId: repo.forge.core, documentType: DOC.packManifest, data, ...(intent ? { intent } : {}) }) + } catch (e) { + if (!isDuplicate(e)) throw e + const id = await findOwnManifest(sdk, repo, auth.identityId, input.packHash) + if (id === null) throw e + return alreadyRecorded(id) + } +} + +/** One chunk: its seq and up to three ≤ 4,900-byte fields (forge-core `pack::split`). */ +export interface Chunk { + readonly seq: number + readonly fields: readonly Uint8Array[] +} + +/** Split `bytes` into chunks exactly as forge-core `pack::split` does. */ +export function splitChunks(bytes: Uint8Array): Chunk[] { + const chunks: Chunk[] = [] + let fields: Uint8Array[] = [] + for (let at = 0; at < bytes.length; at += FIELD_MAX) { + fields.push(bytes.subarray(at, at + FIELD_MAX)) + if (fields.length === CHUNK_FIELDS) { + chunks.push({ seq: chunks.length, fields }) + fields = [] + } + } + if (fields.length > 0) chunks.push({ seq: chunks.length, fields }) + return chunks +} + +/** The seqs of `packHash` the signer already stored in `repo`. */ +async function storedSeqs(sdk: EvoSDK, repo: V2RepoRef, ownerId: string, packHashHex: string, total: number): Promise> { + const have = new Set() + const source = repoSource(repo) + for (let start = 0; start < total; start += 100) { + const seqs = Array.from({ length: Math.min(100, total - start) }, (_, i) => start + i) + const { documents } = await queryDocumentsWithProof(sdk, source.chunkQuery(packHashHex, ownerId, seqs)) + for (const d of documents) { + const raw = d['seq'] + const n = typeof raw === 'bigint' ? Number(raw) : typeof raw === 'number' ? raw : -1 + if (n >= 0) have.add(n) + } + } + return have +} + +/** + * Store `bytes` as the signer's `chunk` documents in `repo` (resumable: seqs already on chain + * are skipped, so a retry after an interruption pays only for what is missing). Returns the + * `platform://///` locator a manifest records. + */ +export async function putPlatformChunks( + sdk: EvoSDK, + auth: WriteAuth, + repo: V2RepoRef, + bytes: Uint8Array, + packHashHex: string, + /** `written`: chunks this call itself created (0 on the first report, before any write). */ + onProgress?: (done: number, total: number, written: number) => void, +): Promise<{ locator: string; chunkCount: number }> { + const chunks = splitChunks(bytes) + const have = await storedSeqs(sdk, repo, auth.identityId, packHashHex, chunks.length) + const R = decodeIdentifier(repo.repoId) + const hash = hexToBytes(packHashHex) + let done = have.size + let written = 0 + onProgress?.(done, chunks.length, written) + for (const chunk of chunks) { + if (have.has(chunk.seq)) continue + const data: Record = { repoId: R, packHash: hash, seq: chunk.seq } + chunk.fields.forEach((f, i) => { + data[`d${i}`] = new Uint8Array(f) + }) + try { + await createDocumentIdempotent(sdk, auth, { + contractId: repo.forge.core, + documentType: DOC.chunk, + data, + intent: `chunk:${repo.repoId}:${packHashHex}:${chunk.seq}`, + }) + written += 1 + } catch (e) { + if (!isDuplicate(e)) throw e + } + done += 1 + onProgress?.(done, chunks.length, written) + } + return { locator: `platform://${repo.forge.core}/${repo.repoId}/${auth.identityId}/${packHashHex}`, chunkCount: chunks.length } +} diff --git a/forge-web/lib/sdk/cost.ts b/forge-web/lib/sdk/cost.ts index 6ba8fae17..ead306e6d 100644 --- a/forge-web/lib/sdk/cost.ts +++ b/forge-web/lib/sdk/cost.ts @@ -34,6 +34,10 @@ export const BASE_CREDITS: Readonly> = { label: 45_000_000, refUpdate: 45_000_000, protectedRefUpdate: 45_000_000, + // Not yet measured from the browser: forge-core's per-byte model (fixed shape plus the + // storage of its byte fields, which `textBytes` does not count — see `estimateBytesCredits`). + packManifest: 60_000_000, + chunk: 20_000_000, // forge-collab issue: 57_400_000, patch: 112_000_000, @@ -119,6 +123,21 @@ export function previewCreate(documentType: string, data: Readonly> = {}): number { + return estimateCreateCredits(documentType, data) + CREDITS_PER_TEXT_BYTE * byteLen +} + +/** + * Estimated credits to store `bytes` as Platform `chunk` documents (three 4,900-byte fields + * each, forge-core `pack::split`): every chunk's fixed cost plus the storage of its bytes. + * Roughly 0.28 DASH per MiB — Platform storage is permanent. + */ +export function estimateChunkCredits(bytes: number): number { + const chunks = Math.max(1, Math.ceil(bytes / (4900 * 3))) + return chunks * (BASE_CREDITS['chunk'] ?? DEFAULT_BASE_CREDITS) + CREDITS_PER_TEXT_BYTE * bytes +} + /** Preview for deleting one document of `documentType` (usually a refund: negative credits). */ export function previewDelete(documentType: string): CostPreview { return previewCredits(DELETE_CREDITS[documentType] ?? DEFAULT_DELETE_CREDITS) diff --git a/forge-web/lib/storage/cid.test.ts b/forge-web/lib/storage/cid.test.ts new file mode 100644 index 000000000..693e18490 --- /dev/null +++ b/forge-web/lib/storage/cid.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest' + +import { CID_CHUNK_SIZE, cidV1RawLeaves, isCid } from './cid' + +describe('cidV1RawLeaves (parity with forge-core backends/cid.rs)', () => { + it('derives the well-known empty raw block', () => { + expect(cidV1RawLeaves(new Uint8Array(0))).toBe('bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku') + }) + + it('derives a single raw leaf', () => { + expect(cidV1RawLeaves(new TextEncoder().encode('hello'))).toBe('bafkreibm6jg3ux5qumhcn2b3flc3tyu6dmlb4xa7u5bf44yegnrjhc4yeq') + }) + + it('makes a dag-pb root past one chunk, deterministically', () => { + const data = new Uint8Array(CID_CHUNK_SIZE + 1).fill(7) + const cid = cidV1RawLeaves(data) + expect(cid.startsWith('bafybei')).toBe(true) + expect(cidV1RawLeaves(data)).toBe(cid) + // What `ipfs add --only-hash` returns for these bytes under the pinned parameters. + expect(cid).toBe('bafybeihldj5jykexbnvf5jx3cm2pjlqqa3hwyc5d6wo2y7ujlk7qjjgzlu') + }) + + it('recognizes CIDs and refuses path smuggling', () => { + expect(isCid('bafkreibm6jg3ux5qumhcn2b3flc3tyu6dmlb4xa7u5bf44yegnrjhc4yeq')).toBe(true) + expect(isCid('bafy/../../api')).toBe(false) + expect(isCid('')).toBe(false) + }) +}) diff --git a/forge-web/lib/storage/cid.ts b/forge-web/lib/storage/cid.ts new file mode 100644 index 000000000..a653099d4 --- /dev/null +++ b/forge-web/lib/storage/cid.ts @@ -0,0 +1,121 @@ +/** + * Local CIDv1 derivation for the exact import parameters Forge asks kubo for + * (`cid-version=1&raw-leaves=true&chunker=size-262144&hash=sha2-256`, balanced layout, 174 + * links per node). A port of forge-core `backends/cid.rs`: the CID kubo returns must equal this + * re-derivation, so the CID a manifest records is a second integrity check next to its sha256. + */ + +import { sha256 } from '@noble/hashes/sha2.js' + +/** kubo's fixed-size chunker (`size-262144`). */ +export const CID_CHUNK_SIZE = 262_144 +/** Links per UnixFS node in the balanced layout (go-unixfs `DefaultLinksPerBlock`). */ +export const CID_MAX_LINKS = 174 + +const CODEC_RAW = 0x55 +const CODEC_DAG_PB = 0x70 +const MH_SHA2_256 = 0x12 + +interface Block { + readonly cid: Uint8Array + readonly tsize: number + readonly content: number +} + +function varint(out: number[], v: number): void { + let n = v + for (;;) { + const byte = n % 128 + n = Math.floor(n / 128) + if (n === 0) { + out.push(byte) + return + } + out.push(byte | 0x80) + } +} + +function key(out: number[], field: number, wire: number): void { + varint(out, field * 8 + wire) +} + +function cidBytes(codec: number, block: Uint8Array): Uint8Array { + const out: number[] = [] + varint(out, 1) + varint(out, codec) + varint(out, MH_SHA2_256) + varint(out, 32) + return Uint8Array.from([...out, ...sha256(block)]) +} + +function leaf(chunk: Uint8Array): Block { + return { cid: cidBytes(CODEC_RAW, chunk), tsize: chunk.length, content: chunk.length } +} + +/** A dag-pb UnixFS `File` node linking `children` (links before data, the canonical order). */ +function fileNode(children: readonly Block[]): Block { + const content = children.reduce((s, c) => s + c.content, 0) + const unixfs: number[] = [] + key(unixfs, 1, 0) + varint(unixfs, 2) + key(unixfs, 3, 0) + varint(unixfs, content) + for (const c of children) { + key(unixfs, 4, 0) + varint(unixfs, c.content) + } + const node: number[] = [] + for (const c of children) { + const link: number[] = [] + key(link, 1, 2) + varint(link, c.cid.length) + link.push(...c.cid) + key(link, 2, 2) + varint(link, 0) + key(link, 3, 0) + varint(link, c.tsize) + key(node, 2, 2) + varint(node, link.length) + node.push(...link) + } + key(node, 1, 2) + varint(node, unixfs.length) + node.push(...unixfs) + const bytes = Uint8Array.from(node) + return { cid: cidBytes(CODEC_DAG_PB, bytes), tsize: bytes.length + children.reduce((s, c) => s + c.tsize, 0), content } +} + +function base32(bytes: Uint8Array): string { + const alphabet = 'abcdefghijklmnopqrstuvwxyz234567' + let out = 'b' + let buffer = 0 + let bits = 0 + for (const b of bytes) { + buffer = ((buffer << 8) | b) & 0xffff + bits += 8 + while (bits >= 5) { + bits -= 5 + out += alphabet[(buffer >> bits) & 31] + } + } + if (bits > 0) out += alphabet[(buffer << (5 - bits)) & 31] + return out +} + +/** The CIDv1 (base32, `b…`) kubo produces for `bytes` under Forge's pinned import parameters. */ +export function cidV1RawLeaves(bytes: Uint8Array): string { + let level: Block[] = [] + if (bytes.length === 0) level.push(leaf(bytes)) + for (let i = 0; i < bytes.length; i += CID_CHUNK_SIZE) level.push(leaf(bytes.subarray(i, i + CID_CHUNK_SIZE))) + while (level.length > 1) { + const next: Block[] = [] + for (let i = 0; i < level.length; i += CID_MAX_LINKS) next.push(fileNode(level.slice(i, i + CID_MAX_LINKS))) + level = next + } + return base32((level[0] as Block).cid) +} + +/** Whether `s` is a plausible CID (base-encoded: letters and digits only). */ +export function isCid(s: string): boolean { + return /^[A-Za-z0-9]+$/.test(s) +} diff --git a/forge-web/lib/storage/cors.ts b/forge-web/lib/storage/cors.ts new file mode 100644 index 000000000..62295b85a --- /dev/null +++ b/forge-web/lib/storage/cors.ts @@ -0,0 +1,104 @@ +/** + * The exact CORS configuration to paste for each provider (`ux-dx-spec.md` §3.1 step 3), + * prefilled with the bucket and this app's origin. + * + * Two rules, as in forge-core `storage/cors.rs` plus the write half the browser needs: + * - **read**, any origin: `GET`/`HEAD` with `Range`, exposing `Content-Range`, `Content-Length` + * and `ETag`. The objects are public; every Forge web app (and any mirror of it) reads them. + * - **write**, this app's origin only: `PUT`/`GET`/`HEAD`/`DELETE` with the SigV4 headers, so + * browser pushes and merges can upload. Listed header by header rather than `x-amz-*`, + * because not every provider accepts a wildcard there. + */ + +import { FORGE_RPC_PATHS } from './ipfs' +import type { ProviderId } from './profiles' + +/** The headers a signed browser request sends that are not CORS-safelisted. */ +const SIGNED_HEADERS = ['authorization', 'content-type', 'range', 'x-amz-content-sha256', 'x-amz-date', 'x-amz-security-token'] as const +const EXPOSE = ['Content-Range', 'Content-Length', 'ETag'] as const +const EXPOSE_LOWER = EXPOSE.map((h) => h.toLowerCase()) + +/** A fix: where to apply it, and the text to paste or run. */ +export interface CorsFix { + readonly where: string + readonly text: string +} + +function s3Rules(origin: string): unknown[] { + return [ + { AllowedOrigins: ['*'], AllowedMethods: ['GET', 'HEAD'], AllowedHeaders: ['Range'], ExposeHeaders: [...EXPOSE], MaxAgeSeconds: 86400 }, + { AllowedOrigins: [origin], AllowedMethods: ['PUT', 'GET', 'HEAD', 'DELETE'], AllowedHeaders: [...SIGNED_HEADERS], ExposeHeaders: [...EXPOSE], MaxAgeSeconds: 86400 }, + ] +} + +/** The CORS fix for `provider`, prefilled with `bucket` and the app `origin`. */ +export function corsFix(provider: ProviderId, bucket: string, origin: string): CorsFix { + const b = bucket || '' + switch (provider) { + case 'r2': + return { + where: `Cloudflare dashboard → R2 → ${b} → Settings → CORS Policy → Add CORS policy. Paste:`, + text: JSON.stringify(s3Rules(origin), null, 2), + } + case 'aws': + return { + where: `If the endpoint is not the bucket's own region, AWS answers with a redirect the browser cannot follow, which looks like a CORS failure: check the endpoint first (https://s3..amazonaws.com). Then save as cors.json and run: aws s3api put-bucket-cors --bucket ${b} --cors-configuration file://cors.json`, + text: JSON.stringify({ CORSRules: s3Rules(origin) }, null, 2), + } + case 'b2': + return { + where: `Save as cors.json, then run: b2 bucket update --cors-rules "$(cat cors.json)" ${b} allPublic`, + text: JSON.stringify( + [ + { corsRuleName: 'dashForgeRead', allowedOrigins: ['*'], allowedOperations: ['s3_get', 's3_head', 'b2_download_file_by_name'], allowedHeaders: ['range'], exposeHeaders: EXPOSE_LOWER, maxAgeSeconds: 86400 }, + { corsRuleName: 'dashForgeWrite', allowedOrigins: [origin], allowedOperations: ['s3_put', 's3_get', 's3_head', 's3_delete'], allowedHeaders: [...SIGNED_HEADERS], exposeHeaders: EXPOSE_LOWER, maxAgeSeconds: 86400 }, + ], + null, + 2, + ), + } + case 'minio': + return { + where: + 'MinIO answers CORS for every origin by default (cors_allow_origin="*", which reads need: every Forge web app and mirror reads the objects). If it was restricted, restore it, then make the bucket publicly readable (not writable):', + text: ['mc admin config set api cors_allow_origin="*"', 'mc admin service restart ', `mc anonymous set download /${b}`].join('\n'), + } + case 'kubo': + case 'pinning': + return { + where: + 'kubo refuses RPC calls from web pages unless their origin is allowed. The RPC API is the node’s admin interface, so give this app a token limited to what Forge calls (below), put that token in the profile’s API Authorization field, and add this origin to the allowed list. Run, then restart the daemon:', + text: kuboCorsLines(origin), + } + case 'platform': + return { where: 'Dash Platform needs no CORS setup.', text: '' } + } +} + +/** + * Shell lines that (1) create a kubo API token limited to the paths Forge calls, (2) ADD this + * app's origin to the RPC API's allowed origins without dropping existing ones (IPFS WebUI's, + * for instance), and (3) let any origin read the gateway. `jq` merges the list. + */ +export function kuboCorsLines(origin: string): string { + const paths = JSON.stringify(FORGE_RPC_PATHS) + return [ + '# 1. Once any API.Authorizations entry exists, kubo refuses EVERY RPC call without a token,', + '# your own `ipfs` commands and IPFS WebUI included. So first an owner token with full', + '# access (keep it; use it as `ipfs --api-auth bearer:$OWNER `), then one for this app', + '# that can only add, pin-check, unpin and identify:', + `OWNER=$(openssl rand -hex 24); TOKEN=$(openssl rand -hex 24)`, + `ipfs config --json API.Authorizations.owner "{\\"AuthSecret\\": \\"bearer:$OWNER\\", \\"AllowedPaths\\": [\\"/api/v0\\"]}"`, + `ipfs config --json API.Authorizations.dash-forge "{\\"AuthSecret\\": \\"bearer:$TOKEN\\", \\"AllowedPaths\\": ${paths.replace(/"/g, '\\"')}}"`, + 'echo "Owner token (for your own ipfs CLI): bearer:$OWNER"', + 'echo "API Authorization for the Forge profile: Bearer $TOKEN"', + '# 2. Allow this app, keeping the origins already allowed (none on a fresh node):', + `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin "$( (ipfs config API.HTTPHeaders.Access-Control-Allow-Origin 2>/dev/null || echo null) | jq -c '(. // []) + ["${origin}"] | unique')"`, + `# (no jq? ipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin '["${origin}"]' replaces the list instead)`, + `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Methods '["POST"]'`, + `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Headers '["Authorization"]'`, + '# 3. Let any origin read the gateway (the content is public):', + `ipfs config --json Gateway.HTTPHeaders.Access-Control-Allow-Origin '["*"]'`, + `ipfs config --json Gateway.HTTPHeaders.Access-Control-Expose-Headers '["Content-Range", "Content-Length", "ETag"]'`, + ].join('\n') +} diff --git a/forge-web/lib/storage/index.ts b/forge-web/lib/storage/index.ts new file mode 100644 index 000000000..bca241664 --- /dev/null +++ b/forge-web/lib/storage/index.ts @@ -0,0 +1,55 @@ +/** + * Bring-your-own storage in the browser (`ux-dx-spec.md` §3.1): profiles sealed in the vault, + * the live bucket test, and the upload path browser writes that push packs (merge, fork + * follow-ups, release assets) go through. + */ + +import type { EvoSDK } from '@dashevo/evo-sdk' + +import type { WriteAuth, WriteResult } from '../sdk' +import type { V2RepoRef } from '../repo/contract' +import { writePackManifest, type PackManifestInput } from '../repo/push' +import { storeArtifact, type StoreOptions, type StoredArtifact } from './upload' + +export * from './profiles' +export * from './store' +export { corsFix, type CorsFix } from './cors' +export { probeProfile, IPFS_ROWS, S3_ROWS, type ProbeRow, type RowId, type RowState } from './probe' +export { + PlatformDeclinedError, + ReplicationError, + fitManifestUris, + orderUris, + storeArtifact, + type PlatformQuestion, + type StoreOptions, + type StoredArtifact, + type TargetFailure, + type UploadEvent, +} from './upload' + +/** What {@link storeAndRecordPack} did: where the bytes went and the manifest that records them. */ +export interface RecordedPack { + readonly stored: StoredArtifact + readonly manifest: WriteResult +} + +/** + * Upload an artifact under `policy`, then write its `packManifest` — the two steps every + * browser push does per artifact, in that order (the manifest only after the bytes are stored + * and verified). A failure after the upload leaves the bytes stored; a retry re-verifies them + * (content-addressed keys, CIDs, the signer's own chunks) and writes only the manifest. + */ +export async function storeAndRecordPack( + sdk: EvoSDK, + auth: WriteAuth, + repo: V2RepoRef, + bytes: Uint8Array, + meta: Pick, + opts: StoreOptions & { readonly intent?: string }, +): Promise { + const stored = await storeArtifact(sdk, auth, repo, bytes, opts) + const { packHash, sizeBytes, chunkCount, storage, uris } = stored + const manifest = await writePackManifest(sdk, auth, repo, { ...meta, packHash, sizeBytes, chunkCount, storage, uris }, opts.intent) + return { stored, manifest } +} diff --git a/forge-web/lib/storage/ipfs.ts b/forge-web/lib/storage/ipfs.ts new file mode 100644 index 000000000..caea9ffdb --- /dev/null +++ b/forge-web/lib/storage/ipfs.ts @@ -0,0 +1,196 @@ +/** + * IPFS from the browser: kubo's RPC API (`add` with Forge's pinned import parameters, `pin/ls`, + * `pin/rm`, `id`) and the IPFS Pinning Service API (`POST /pins`, `GET /pins/`). Port of + * forge-core `backends/ipfs.rs`: the CID kubo returns must equal the local re-derivation + * (`./cid`), and the node must report it pinned, before any URI is recorded. + * + * kubo's RPC API refuses cross-origin calls unless `API.HTTPHeaders.Access-Control-Allow-Origin` + * lists this origin (it answers 403 otherwise); the wizard prints the `ipfs config` lines. + * + * The RPC API is kubo's ADMIN interface. Allowing a web origin to call it hands that origin + * the node, unless kubo's `API.Authorizations` limits the token it uses to the few paths Forge + * needs ({@link FORGE_RPC_PATHS}); the wizard says so and prints that configuration, and the + * token goes into the profile's `apiAuth`. + */ + +import { cidV1RawLeaves, isCid } from './cid' +import type { ProfilePublic, ProfileSecrets } from './profiles' +import { TimeoutError, isHeaderSafe, timedFetch } from './util' + +export type IpfsSettings = Extract + +/** The kubo `add` query pinning every import parameter the local CID derivation assumes. */ +export const ADD_PARAMS = + 'cid-version=1&raw-leaves=true&chunker=size-262144&hash=sha2-256&trickle=false&max-file-links=174&pin=true&quieter=true' + +/** The RPC paths Forge calls: what a restricted `API.Authorizations` entry must allow. */ +export const FORGE_RPC_PATHS = ['/api/v0/add', '/api/v0/pin/ls', '/api/v0/pin/rm', '/api/v0/id', '/api/v0/version'] as const + +const PIN_TIMEOUT_MS = 120_000 +const PIN_POLL_MS = 2_000 + +export class IpfsError extends Error { + constructor( + readonly op: string, + message: string, + ) { + super(message) + this.name = 'IpfsError' + } +} + +async function rpc(s: IpfsSettings, secrets: ProfileSecrets, op: string, pathAndQuery: string, body?: FormData, uploadBytes = 0): Promise { + const headers = new Headers() + if (secrets.apiAuth) { + // A value fetch refuses would be quoted in its error: refuse it here, without echoing it. + if (!isHeaderSafe(secrets.apiAuth)) throw new IpfsError(op, 'the API Authorization value holds characters an HTTP header cannot carry; paste it again') + headers.set('authorization', secrets.apiAuth) + } + const url = `${s.api.replace(/\/+$/, '')}/api/v0/${pathAndQuery}` + let text: string + let resp: Response + try { + const r = await timedFetch(url, { method: 'POST', headers, ...(body ? { body } : {}), redirect: 'error', credentials: 'omit' }, { uploadBytes }) + resp = r.resp + text = await r.text() + } catch (e) { + const why = e instanceof TimeoutError ? e.message : 'unreachable, or it refused this origin (CORS)' + throw new IpfsError(op, `${op}: the kubo API at ${new URL(url).host}: ${why}`) + } + if (!resp.ok) { + const msg = /"Message"\s*:\s*"([^"]{1,200})"/.exec(text)?.[1] ?? text.slice(0, 200) + const hint = + resp.status === 403 + ? ' — kubo refuses this origin: allow it in API.HTTPHeaders (see the fix below)' + : resp.status === 401 + ? ' — kubo wants an API Authorization token (see API.Authorizations)' + : '' + throw new IpfsError(op, `${op} failed: HTTP ${resp.status} ${msg}${hint}`) + } + return text +} + +/** Let kubo's "not pinned" answer through as '' (nothing pinned); rethrow anything else. */ +function ignoreNotPinned(e: unknown): string { + if (e instanceof IpfsError && /not pinned/.test(e.message)) return '' + throw e +} + +/** kubo's version (`/api/v0/version`), a cheap reachability check. */ +export async function kuboVersion(s: IpfsSettings, secrets: ProfileSecrets): Promise { + const text = await rpc(s, secrets, 'version', 'version') + return /"Version"\s*:\s*"([^"]+)"/.exec(text)?.[1] ?? 'unknown' +} + +/** Add `bytes` with the pinned parameters; the CID must equal the local derivation and be pinned. */ +export async function addVerified(s: IpfsSettings, secrets: ProfileSecrets, bytes: Uint8Array): Promise { + const form = new FormData() + form.append('file', new Blob([new Uint8Array(bytes)]), 'pack') + const text = await rpc(s, secrets, 'ipfs add', `add?${ADD_PARAMS}`, form, bytes.length) + const line = text.trim().split('\n').pop() ?? '' + const cid = /"Hash"\s*:\s*"([^"]+)"/.exec(line)?.[1] ?? '' + const expected = cidV1RawLeaves(bytes) + if (cid !== expected) { + throw new IpfsError('ipfs add', `kubo returned CID ${cid || '(none)'} but these bytes derive to ${expected} under the pinned import parameters; refusing to record it`) + } + const pinned = await rpc(s, secrets, 'pin check', `pin/ls?arg=${cid}&type=recursive`).catch(ignoreNotPinned) + if (!pinned.includes(cid)) throw new IpfsError('pin check', `kubo added ${cid} but does not report it pinned; a gc would drop it`) + return cid +} + +/** Unpin `cid` (the wizard's probe cleanup). A CID that is not pinned is fine. */ +export async function unpin(s: IpfsSettings, secrets: ProfileSecrets, cid: string): Promise { + if (!isCid(cid)) throw new IpfsError('unpin', 'not a CID') + await rpc(s, secrets, 'unpin', `pin/rm?arg=${cid}`).catch(ignoreNotPinned) +} + +/** The node's announced multiaddrs, handed to a pinning service as `origins`. */ +async function origins(s: IpfsSettings, secrets: ProfileSecrets): Promise { + try { + const text = await rpc(s, secrets, 'id', 'id') + const parsed: unknown = JSON.parse(text) + const addrs = parsed && typeof parsed === 'object' ? (parsed as { Addresses?: unknown }).Addresses : undefined + return Array.isArray(addrs) ? addrs.filter((a): a is string => typeof a === 'string').slice(0, 20) : [] + } catch { + return [] + } +} + +interface PinStatus { + readonly requestid: string + readonly status: string +} + +async function psa(s: IpfsSettings, secrets: ProfileSecrets, op: string, path: string, init: RequestInit = {}): Promise { + if (!secrets.pinningToken) throw new IpfsError(op, 'the pinning service needs an access token') + if (!isHeaderSafe(secrets.pinningToken)) throw new IpfsError(op, 'the pinning token holds characters an HTTP header cannot carry; paste it again') + const url = `${s.pinningEndpoint.replace(/\/+$/, '')}${path}` + let resp: Response + let text: string + try { + const r = await timedFetch(url, { + ...init, + headers: { authorization: `Bearer ${secrets.pinningToken}`, 'content-type': 'application/json' }, + redirect: 'error', + credentials: 'omit', + }) + resp = r.resp + text = await r.text() + } catch (e) { + const why = e instanceof TimeoutError ? e.message : 'unreachable, or it refused this origin (CORS)' + throw new IpfsError(op, `${op}: the pinning service: ${why}`) + } + if (!resp.ok) { + const hint = resp.status === 401 || resp.status === 403 ? ' — check the access token' : '' + throw new IpfsError(op, `${op} failed: HTTP ${resp.status}${hint}`) + } + try { + return JSON.parse(text) as unknown + } catch { + throw new IpfsError(op, `${op}: the pinning service did not answer with JSON`) + } +} + +function pinStatus(v: unknown): PinStatus { + const o = v && typeof v === 'object' ? (v as Record) : {} + const requestid = typeof o['requestid'] === 'string' ? o['requestid'] : '' + const status = typeof o['status'] === 'string' ? o['status'] : '' + if (!/^[A-Za-z0-9_-]{1,128}$/.test(requestid)) throw new IpfsError('pin', 'the pinning service returned a malformed request id') + return { requestid, status } +} + +/** An authenticated no-op against the pinning service (`GET /pins?limit=1`). */ +export async function pinningReachable(s: IpfsSettings, secrets: ProfileSecrets): Promise { + await psa(s, secrets, 'pinning service', '/pins?limit=1') +} + +/** + * Pin `cid` on the pinning service and wait for `pinned`. An existing request for the CID is + * reused (a re-push pays nothing twice). + */ +export async function remotePin(s: IpfsSettings, secrets: ProfileSecrets, cid: string, name: string): Promise { + if (!isCid(cid)) throw new IpfsError('pin', 'not a CID') + const found = await psa(s, secrets, 'pin lookup', `/pins?cid=${cid}&status=queued,pinning,pinned`) + const results = found && typeof found === 'object' ? (found as { results?: unknown }).results : undefined + const existing = Array.isArray(results) ? results.map(pinStatus) : [] + if (existing.some((p) => p.status === 'pinned')) return + let pin = existing.find((p) => p.status === 'queued' || p.status === 'pinning') + if (!pin) { + pin = pinStatus(await psa(s, secrets, 'pin', '/pins', { method: 'POST', body: JSON.stringify({ cid, name, origins: await origins(s, secrets) }) })) + } + const deadline = Date.now() + PIN_TIMEOUT_MS + for (;;) { + if (pin.status === 'pinned') return + if (pin.status === 'failed') throw new IpfsError('pin', `the pinning service reports request ${pin.requestid} failed`) + if (Date.now() >= deadline) { + throw new IpfsError('pin', `the pin is still ${pin.status} after ${PIN_TIMEOUT_MS / 1000}s; the service may not be able to reach your kubo node`) + } + await new Promise((r) => setTimeout(r, PIN_POLL_MS)) + pin = pinStatus(await psa(s, secrets, 'pin status', `/pins/${pin.requestid}`)) + } +} + +/** The gateway URL of `cid` on `gateway`. */ +export function gatewayUrl(gateway: string, cid: string): string { + return `${gateway.replace(/\/+$/, '')}/ipfs/${cid}` +} diff --git a/forge-web/lib/storage/probe.ts b/forge-web/lib/storage/probe.ts new file mode 100644 index 000000000..de3347bf6 --- /dev/null +++ b/forge-web/lib/storage/probe.ts @@ -0,0 +1,288 @@ +/** + * The storage wizard's live test (`ux-dx-spec.md` §3.1 step 3), run from this page, so it + * checks exactly what the browser will do later — including CORS, which a CLI test cannot see. + * + * S3 rows: signed PUT → signed GET → anonymous GET via the public URL → ranged read with + * `Content-Range` visible → CORS for browser pushes (PUT) → delete probe. + * IPFS rows: kubo API → add + CID + pin → gateway re-read (Range) → public gateway → + * pinning service → unpin. + * + * A page cannot see WHY a cross-origin request failed: a CORS refusal, a refused redirect and a + * dead host all reject `fetch`. {@link reachable} tells a dead host apart with an opaque + * `no-cors` request to the same origin, which resolves whenever the host answers at all; a + * `manual`-redirect probe then tells a redirect (wrong region or endpoint) from CORS. + * + * The public address is what everyone else reads, and it is recorded on chain: a row fails + * when it is only reachable from this machine or its network, even if it answers here. + */ + +import { S3Error, deleteObject, getObject, getPublic, objectUrl, publicObjectUrl, putObject, type S3Settings } from './s3' +import { IpfsError, addVerified, gatewayUrl, kuboVersion, pinningReachable, unpin, type IpfsSettings } from './ipfs' +import { normalizedPrefix, publishProblem, type ProfileSecrets, type StorageProfile } from './profiles' +import { sha256Hex } from './sigv4' +import { bytesEqual, errText, timedFetch } from './util' + +export type RowId = 'put' | 'get' | 'public' | 'range' | 'cors-put' | 'delete' | 'api' | 'add' | 'gateway' | 'public-gateway' | 'pinning' | 'unpin' +export type RowState = 'pending' | 'running' | 'ok' | 'fail' | 'skipped' + +/** One row of the live test. `cors` marks a failure the CORS fix block addresses. */ +export interface ProbeRow { + readonly id: RowId + readonly label: string + readonly state: RowState + readonly detail: string + readonly cors?: boolean +} + +export const S3_ROWS: readonly { id: RowId; label: string }[] = [ + { id: 'put', label: 'signed PUT' }, + { id: 'get', label: 'signed GET' }, + { id: 'public', label: 'anonymous GET via public URL' }, + { id: 'range', label: 'ranged read (Content-Range visible)' }, + { id: 'cors-put', label: 'CORS preflight (PUT) for browser pushes' }, + { id: 'delete', label: 'delete probe' }, +] + +export const IPFS_ROWS: readonly { id: RowId; label: string }[] = [ + { id: 'api', label: 'kubo API' }, + { id: 'add', label: 'add + CID check + pin' }, + { id: 'gateway', label: 'gateway re-read (Range)' }, + { id: 'public-gateway', label: 'anonymous read via public gateway' }, + { id: 'pinning', label: 'pinning service' }, + { id: 'unpin', label: 'unpin probe' }, +] + +/** Whether `url`'s origin answers at all (an opaque no-cors request; never readable). */ +async function reachable(url: string | URL): Promise { + try { + await fetch(new URL(url).origin, { mode: 'no-cors', credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(10_000) }) + return true + } catch { + return false + } +} + +/** + * Whether `url` answers with a redirect. A redirect with a `Location` shows as an opaque + * redirect; AWS's wrong-region answer is a bare `301 PermanentRedirect` with no `Location`, + * which fetch hands back as an ordinary response, visible only through a CORS read of its + * status (a `no-cors` one is opaque) — so both are tried. + */ +async function redirects(url: string | URL): Promise { + const opts = { redirect: 'manual' as const, credentials: 'omit' as const, cache: 'no-store' as const } + try { + const r = await fetch(url, { ...opts, mode: 'no-cors', signal: AbortSignal.timeout(10_000) }) + if (r.type === 'opaqueredirect') return true + } catch { + return false + } + try { + const r = await fetch(url, { ...opts, signal: AbortSignal.timeout(10_000) }) + return r.type === 'opaqueredirect' || (r.status >= 300 && r.status < 400) + } catch { + return false + } +} + +/** Why a request to `url` never got an answer: down, redirected elsewhere, or CORS. */ +async function whyBlocked(url: string | URL): Promise<'down' | 'redirect' | 'cors'> { + if (!(await reachable(url))) return 'down' + return (await redirects(url)) ? 'redirect' : 'cors' +} + +/** A unique probe body, so a stale cached object can never pass for this run's upload. */ +function probeBody(): Uint8Array { + const nonce = Array.from(crypto.getRandomValues(new Uint8Array(12)), (b) => b.toString(16).padStart(2, '0')).join('') + return new TextEncoder().encode(`dash-forge web storage probe ${Date.now()} ${nonce}\n`) +} + +type Report = (id: RowId, state: RowState, detail: string, cors?: boolean) => void + +/** Run the S3 test, reporting each row as it settles. */ +async function probeS3(p: StorageProfile & { settings: S3Settings }, report: Report): Promise { + const s = p.settings + const secrets = p.secrets + const body = probeBody() + const key = `${normalizedPrefix(s.prefix)}probe/forge-web-test-${await sha256Hex(body)}.txt` + const apiUrl = objectUrl(s, key) + + // 1 + 5: a PUT that lands proves the PUT preflight passed; one refused before any answer is + // CORS when the bucket's host answers and does not redirect. + report('put', 'running', '') + try { + await putObject(s, secrets, key, body, 'text/plain') + report('put', 'ok', `wrote ${key}`) + report('cors-put', 'ok', 'the browser was allowed to send a signed PUT from this origin') + } catch (e) { + const why = e instanceof S3Error && e.status === 0 ? await whyBlocked(apiUrl) : null + if (why === 'cors') { + report('put', 'fail', 'the browser refused to send it (see the CORS row)') + report('cors-put', 'fail', 'the bucket’s CORS rules do not allow a signed PUT from this origin', true) + } else if (why === 'redirect') { + report('put', 'fail', `${apiUrl.host} answered with a redirect: the bucket lives in another region or behind another endpoint; set that endpoint`) + report('cors-put', 'skipped', 'needs a PUT that reaches the bucket') + } else if (why === 'down') { + report('put', 'fail', `${apiUrl.host} did not answer (check the endpoint${s.pathStyle ? '' : ', and that the bucket subdomain resolves'})`) + report('cors-put', 'skipped', 'needs a PUT that reaches the bucket') + } else { + report('put', 'fail', errText(e)) + report('cors-put', 'skipped', 'needs a PUT that reaches the bucket') + } + for (const id of ['get', 'public', 'range', 'delete'] as const) report(id, 'skipped', 'needs the probe object') + return + } + + report('get', 'running', '') + try { + const got = await getObject(s, secrets, key) + if (bytesEqual(got, body)) report('get', 'ok', 'read back byte for byte') + else report('get', 'fail', 'the bucket returned different bytes (a cache or proxy in front of it?)') + } catch (e) { + report('get', 'fail', errText(e)) + } + + const publicUrl = publicObjectUrl(s, key) + const publicHost = new URL(publicUrl).host + const unpublishable = publishProblem(p) + report('public', 'running', '') + // Whether an anonymous read worked from here (the range row builds on it), apart from whether + // the address is one others can use (a local MinIO answers here and nowhere else). + let publicOk = false + try { + const got = await getPublic(s, key) + if (got.status !== 200) report('public', 'fail', `HTTP ${got.status} from ${publicHost}: the objects are not publicly readable`) + else if (!bytesEqual(got.bytes, body)) report('public', 'fail', `${publicHost} served different bytes (check the public URL)`) + else { + publicOk = true + if (unpublishable) report('public', 'fail', `Readable from here, but ${unpublishable}`) + else report('public', 'ok', `anonymous GET ${publicHost}`) + } + } catch { + const why = await whyBlocked(publicUrl) + report( + 'public', + 'fail', + why === 'cors' ? 'the browser was not allowed to read it: no Access-Control-Allow-Origin on GET' : why === 'redirect' ? `${publicHost} redirected the read elsewhere` : `${publicHost} did not answer`, + why === 'cors', + ) + } + + report('range', 'running', '') + if (!publicOk) report('range', 'skipped', 'needs a public read') + else { + try { + const got = await getPublic(s, key, 'bytes=0-9') + if (got.status !== 206) report('range', 'fail', `a ranged GET returned ${got.status} instead of 206: browsing needs HTTP Range support on the public URL`) + else if (got.contentRange === null) report('range', 'fail', 'Content-Range is not exposed to scripts (Access-Control-Expose-Headers)', true) + else report('range', 'ok', 'Range honoured, Content-Range exposed') + } catch { + report('range', 'fail', 'the ranged read was refused: allow the Range request header in the bucket’s CORS rules', true) + } + } + + report('delete', 'running', '') + try { + await deleteObject(s, secrets, key) + report('delete', 'ok', 'probe removed') + } catch (e) { + const cors = e instanceof S3Error && e.status === 0 && (await whyBlocked(apiUrl)) === 'cors' + report('delete', 'fail', `${errText(e)}. The probe object (a few bytes) stays at ${key}; delete it by hand.`, cors) + } +} + +/** Read `url` anonymously and require exactly `body`: a row's verdict. */ +async function readBack(url: string, body: Uint8Array, range: boolean): Promise<{ ok: true } | { ok: false; detail: string; cors: boolean }> { + try { + const r = await timedFetch(url, { ...(range ? { headers: { Range: `bytes=0-${body.length - 1}` } } : {}), credentials: 'omit', cache: 'no-store' }) + const got = await r.bytes() + if (!r.resp.ok) return { ok: false, detail: `HTTP ${r.resp.status}`, cors: false } + if (!bytesEqual(got, body)) return { ok: false, detail: 'served different bytes', cors: false } + return { ok: true } + } catch { + const why = await whyBlocked(url) + return { ok: false, detail: why === 'cors' ? 'refused this origin (CORS)' : why === 'redirect' ? 'redirected elsewhere (use 127.0.0.1, not localhost, for a local gateway)' : 'did not answer', cors: why === 'cors' } + } +} + +/** Run the IPFS test (kubo, plus the pinning service for a pinning profile). */ +async function probeIpfs(p: StorageProfile & { settings: IpfsSettings }, report: Report): Promise { + const s = p.settings + const secrets: ProfileSecrets = p.secrets + report('api', 'running', '') + try { + report('api', 'ok', `kubo ${await kuboVersion(s, secrets)}`) + } catch (e) { + const why = await whyBlocked(s.api) + report('api', 'fail', why === 'down' ? `${new URL(s.api).host} did not answer` : errText(e), why === 'cors') + for (const id of ['add', 'gateway', 'public-gateway', 'pinning', 'unpin'] as const) report(id, 'skipped', 'needs the kubo API') + return + } + + const body = probeBody() + let cid = '' + report('add', 'running', '') + try { + cid = await addVerified(s, secrets, body) + report('add', 'ok', `${cid} matches the local derivation and is pinned`) + } catch (e) { + report('add', 'fail', errText(e)) + } + + report('gateway', 'running', '') + if (cid === '' || s.gateway === '') report('gateway', 'skipped', s.gateway === '' ? 'no gateway configured: uploads are verified by CID and pin only' : 'needs the probe') + else { + const r = await readBack(gatewayUrl(s.gateway, cid), body, true) + if (r.ok) report('gateway', 'ok', `re-read through ${new URL(s.gateway).host}`) + else report('gateway', 'fail', `the gateway ${r.detail}`, r.cors) + } + + report('public-gateway', 'running', '') + if (s.publicGateway === '') report('public-gateway', 'skipped', 'no public gateway: readers race public IPFS gateways, which may not find content only your node holds') + else if (cid === '') report('public-gateway', 'skipped', 'needs the probe') + else { + const unpublishable = publishProblem(p) + const r = await readBack(gatewayUrl(s.publicGateway, cid), body, false) + if (!r.ok) report('public-gateway', 'fail', `the public gateway ${r.detail}`, r.cors) + else if (unpublishable) report('public-gateway', 'fail', `Readable from here, but ${unpublishable}`) + else report('public-gateway', 'ok', `anonymous read through ${new URL(s.publicGateway).host}`) + } + + report('pinning', 'running', '') + if (s.kind !== 'ipfs-pinning-service') report('pinning', 'skipped', 'no pinning service in this profile') + else { + try { + await pinningReachable(s, secrets) + report('pinning', 'ok', 'the service accepted the token') + } catch (e) { + report('pinning', 'fail', errText(e), e instanceof IpfsError && /CORS/.test(e.message)) + } + } + + report('unpin', 'running', '') + if (cid === '') report('unpin', 'skipped', 'nothing was added') + else { + try { + await unpin(s, secrets, cid) + report('unpin', 'ok', 'probe unpinned (a gc removes it)') + } catch (e) { + report('unpin', 'fail', errText(e)) + } + } +} + +/** + * Run the test for any profile kind, reporting each row as it settles. Resolves with whether + * every row passed (no row failed). A Platform profile needs none. + */ +export async function probeProfile(p: StorageProfile, report: Report): Promise { + const s = p.settings + if (s.kind === 'platform') return true + let ok = true + const tracked: Report = (id, state, detail, cors) => { + if (state === 'fail') ok = false + report(id, state, detail, cors) + } + if (s.kind === 's3') await probeS3({ ...p, settings: s }, tracked) + else await probeIpfs({ ...p, settings: s }, tracked) + return ok +} diff --git a/forge-web/lib/storage/profiles.ts b/forge-web/lib/storage/profiles.ts new file mode 100644 index 000000000..6b695b59b --- /dev/null +++ b/forge-web/lib/storage/profiles.ts @@ -0,0 +1,399 @@ +/** + * Storage profiles in the browser (`ux-dx-spec.md` §3.1): the user's own buckets and IPFS + * nodes, described with the same fields forge-core's `storage.toml` profiles use (camel-cased), + * so a profile means the same thing on both sides. + * + * A profile splits into its public part (endpoint, bucket, public URL: the manifest publishes + * some of it anyway) and its secrets (access key id, secret key, session token, pinning token, + * kubo API auth). Only the public part is ever held in plaintext; the secrets are sealed in + * the vault (`./store`). Nothing here is written on-chain except the public read URLs a push + * records in its `packManifest`. + */ + +import { z } from 'zod' + +import { isPublicHttpsUrl } from '../net' +import { isHeaderSafe } from './util' + +/** The kinds a profile can be (parity with forge-core `Profile`). */ +export type ProfileKind = 's3' | 'ipfs-kubo' | 'ipfs-pinning-service' | 'platform' + +/** The provider tiles of the wizard, in the order the spec lists them. */ +export type ProviderId = 'r2' | 'b2' | 'aws' | 'minio' | 'kubo' | 'pinning' | 'platform' + +/** A profile's public settings. */ +export type ProfilePublic = + | { + readonly kind: 's3' + readonly provider: 'r2' | 'b2' | 'aws' | 'minio' + /** API endpoint origin (`https://.r2.cloudflarestorage.com`). */ + readonly endpoint: string + readonly region: string + readonly bucket: string + /** Path-style addressing (`endpoint/bucket/key`); R2, B2 and MinIO want it. */ + readonly pathStyle: boolean + /** Public read origin; objects are read at `/`. */ + readonly publicUrl: string + /** Key prefix inside the bucket (`forge/`), or ''. */ + readonly prefix: string + } + | { + readonly kind: 'ipfs-kubo' | 'ipfs-pinning-service' + readonly provider: 'kubo' | 'pinning' + /** kubo RPC API origin (`http://127.0.0.1:5001`). */ + readonly api: string + /** A gateway serving this node's content, for the verification re-read. */ + readonly gateway: string + /** A public https gateway to record next to `ipfs://` (optional). */ + readonly publicGateway: string + /** Pinning Service API base URL (pinning profiles only). */ + readonly pinningEndpoint: string + } + | { readonly kind: 'platform'; readonly provider: 'platform' } + +/** The secrets a profile needs, sealed in the vault. */ +export interface ProfileSecrets { + readonly accessKeyId?: string + readonly secretAccessKey?: string + readonly sessionToken?: string + /** The full `Authorization` header value for a kubo RPC API behind auth. */ + readonly apiAuth?: string + /** A Pinning Service API access token. */ + readonly pinningToken?: string +} + +/** A named profile: public settings plus (when unlocked) its secrets. */ +export interface StorageProfile { + readonly name: string + readonly settings: ProfilePublic + readonly secrets: ProfileSecrets +} + +/** One provider tile: what the form asks for and where to find it. */ +export interface ProviderPreset { + readonly id: ProviderId + readonly title: string + readonly blurb: string + readonly kind: ProfileKind + /** Defaults the form starts from. */ + readonly defaults: Partial> + /** Field → "where to find this" hint (with a console link where one exists). */ + readonly hints: Readonly> +} + +export const PROVIDERS: readonly ProviderPreset[] = [ + { + id: 'r2', + title: 'Cloudflare R2', + blurb: 'recommended: free egress', + kind: 's3', + defaults: { region: 'auto', pathStyle: true, endpoint: 'https://.r2.cloudflarestorage.com' }, + hints: { + endpoint: { text: 'R2 → Overview → Account details → S3 API. Use the origin only, without the bucket.', href: 'https://dash.cloudflare.com/?to=/:account/r2/overview' }, + bucket: { text: 'R2 → Create bucket, e.g. forge.' }, + publicUrl: { text: 'Bucket → Settings → Public access: the r2.dev subdomain or your custom domain.' }, + accessKeyId: { text: 'R2 → Manage R2 API Tokens → Create API token (Object Read & Write, this bucket).', href: 'https://dash.cloudflare.com/?to=/:account/r2/api-tokens' }, + region: { text: 'R2 always uses auto.' }, + }, + }, + { + id: 'b2', + title: 'Backblaze B2', + blurb: 'cheap storage, S3 API', + kind: 's3', + defaults: { region: 'us-west-004', pathStyle: true, endpoint: 'https://s3.us-west-004.backblazeb2.com' }, + hints: { + endpoint: { text: 'Buckets → your bucket → Endpoint. The region is its middle part.', href: 'https://secure.backblaze.com/b2_buckets.htm' }, + bucket: { text: 'Create the bucket with Files in bucket: Public.' }, + publicUrl: { text: 'https://s3..backblazeb2.com/ works for public buckets.' }, + accessKeyId: { text: 'App Keys → Add a New Application Key (Read and Write, this bucket). keyID is the access key id.', href: 'https://secure.backblaze.com/app_keys.htm' }, + }, + }, + { + id: 'aws', + title: 'AWS S3', + blurb: 'the original', + kind: 's3', + defaults: { region: 'us-east-1', pathStyle: false, endpoint: 'https://s3.us-east-1.amazonaws.com' }, + hints: { + endpoint: { text: 'https://s3..amazonaws.com for the bucket’s region.', href: 'https://s3.console.aws.amazon.com/s3/buckets' }, + bucket: { text: 'A bucket name with dots needs path-style addressing over TLS.' }, + publicUrl: { text: 'https://.s3..amazonaws.com with a public-read bucket policy, or a CloudFront URL.' }, + accessKeyId: { text: 'IAM → a user or role limited to s3:PutObject, GetObject, DeleteObject on this bucket.', href: 'https://console.aws.amazon.com/iam/home#/users' }, + }, + }, + { + id: 'minio', + title: 'MinIO / other S3', + blurb: 'self-hosted or any S3-compatible store', + kind: 's3', + defaults: { region: 'us-east-1', pathStyle: true, endpoint: 'https://minio.example.org' }, + hints: { + endpoint: { text: 'Your server’s S3 API origin (local MinIO: http://127.0.0.1:9000).' }, + bucket: { text: 'mc mb /forge, then mc anonymous set download /forge.' }, + publicUrl: { text: 'Usually /.' }, + accessKeyId: { text: 'A MinIO user or access key with read and write on the bucket.' }, + }, + }, + { + id: 'kubo', + title: 'IPFS (kubo)', + blurb: 'your own IPFS node', + kind: 'ipfs-kubo', + defaults: { api: 'http://127.0.0.1:5001', gateway: 'http://127.0.0.1:8080' }, + hints: { + api: { text: 'The kubo RPC API (Addresses.API). The browser calls it directly, so it needs CORS for this origin.' }, + gateway: { text: 'A gateway serving this node’s content (Addresses.Gateway), used to re-read uploads.' }, + publicGateway: { text: 'Optional: a public https gateway for your node, recorded so browsers can fetch directly.' }, + }, + }, + { + id: 'pinning', + title: 'IPFS pinning service', + blurb: 'kubo + a remote pin', + kind: 'ipfs-pinning-service', + defaults: { api: 'http://127.0.0.1:5001', gateway: 'http://127.0.0.1:8080' }, + hints: { + api: { text: 'The kubo node the content is added through; the service fetches it from there.' }, + pinningEndpoint: { text: 'The service’s Pinning Service API base URL (for example Pinata’s https://api.pinata.cloud/psa).', href: 'https://ipfs.github.io/pinning-services-api-spec/' }, + pinningToken: { text: 'An access token from the service’s dashboard.' }, + }, + }, + { + id: 'platform', + title: 'Dash Platform', + blurb: 'permanent, ~0.28 DASH per MiB', + kind: 'platform', + defaults: {}, + hints: {}, + }, +] as const + +/** The preset for `id`. */ +export function providerPreset(id: ProviderId): ProviderPreset { + const p = PROVIDERS.find((x) => x.id === id) + if (!p) throw new Error(`unknown provider ${id}`) + return p +} + +/** A valid profile name: it becomes part of a policy list, so no commas or whitespace. */ +export function validProfileName(name: string): boolean { + return /^[A-Za-z0-9._-]{1,64}$/.test(name) +} + +/** The profile name reserved for on-chain `chunk` storage (forge-core `PLATFORM_PROFILE`). */ +export const PLATFORM_PROFILE = 'platform' + +/** + * Where a URL is used: + * - `local`: only this browser talks to it (the S3 API endpoint, the kubo API and gateway). + * https, or plain http to this machine. + * - `published`: recorded on chain, read by everyone (the public URL, the public gateway). + * A public https URL only (`lib/net.ts`): a loopback or private address would be a copy only + * its uploader can read, and would point every reader's browser at their own local services. + */ +type UrlUse = 'local' | 'published' + +/** An http(s) origin or URL with no userinfo, query or fragment (they could smuggle a token). */ +function checkUrl(field: string, value: string, use: UrlUse, opts: { originOnly?: boolean } = {}): string | null { + let url: URL + try { + url = new URL(value) + } catch { + return `${field} is not a URL` + } + if (url.protocol !== 'https:' && url.protocol !== 'http:') return `${field} must be an http(s) URL` + if (url.username || url.password) return `${field} must not carry a user name or password` + if (url.search || url.hash) return `${field} must not carry a query or fragment` + if (opts.originOnly && url.pathname !== '/' && url.pathname !== '') return `${field} must be an origin (scheme://host[:port]) with no path` + // A published URL may point at this machine while testing (a local MinIO): the test runs and + // its public row fails with the reason ({@link publishProblem}); uploads refuse it. + if (url.protocol === 'http:' && !isLoopback(url.hostname)) { + return use === 'published' + ? `${field} must be https: it is recorded on chain for everyone to read` + : `${field} must be https (plain http is allowed only for a node on this machine, at 127.0.0.1 or localhost)` + } + return null +} + +/** Whether a hostname is this machine (the only place a browser may reach over plain http). */ +export function isLoopback(hostname: string): boolean { + return hostname === 'localhost' || hostname === '127.0.0.1' +} + +/** Why a profile's secrets cannot be used, or null (header values must be printable ASCII). */ +function secretsProblem(secrets: ProfileSecrets): string | null { + for (const [field, label] of [ + ['accessKeyId', 'The access key id'], + ['sessionToken', 'The session token'], + ['apiAuth', 'The API Authorization value'], + ['pinningToken', 'The pinning token'], + ] as const) { + const v = secrets[field] + if (v !== undefined && !isHeaderSafe(v)) return `${label} holds characters an HTTP header cannot carry (a stray line break or a non-ASCII character); paste it again` + } + return null +} + +/** Why a profile is not usable, or null. Parity with forge-core `S3Config::validate`. */ +export function profileProblem(p: StorageProfile): string | null { + if (!validProfileName(p.name)) return "use a name of letters, digits, '.', '_' and '-' (max 64)" + const s = p.settings + if (s.kind === 'platform') return null + if (p.name === PLATFORM_PROFILE) return `the name "${PLATFORM_PROFILE}" is reserved for Dash Platform storage` + const secrets = secretsProblem(p.secrets) + if (secrets) return secrets + if (s.kind === 's3') { + const e = checkUrl('the endpoint', s.endpoint, 'local', { originOnly: true }) + if (e) return e + if (!BUCKET.test(s.bucket)) return "the bucket name must be lowercase letters, digits, '-', '.', '_'" + const host = new URL(s.endpoint).hostname + if (!s.pathStyle && (/^\d+\.\d+\.\d+\.\d+$/.test(host) || host.startsWith('['))) return 'an IP-address endpoint needs path-style addressing' + if (!s.pathStyle && s.bucket.includes('.') && s.endpoint.startsWith('https:')) return "a bucket name with '.' breaks TLS for virtual-hosted addressing; use path-style" + const prefix = s.prefix.replace(/\/+$/, '') + if (prefix !== '' && prefix.split('/').some((seg) => seg === '' || seg === '.' || seg === '..')) return "the prefix has an empty, '.' or '..' segment" + if (s.publicUrl === '') return 'a public URL is needed: browsers read packs anonymously from it' + const pu = checkUrl('the public URL', s.publicUrl, 'published') + if (pu) return pu + if (!p.secrets.accessKeyId || !p.secrets.secretAccessKey) return 'an access key id and secret are needed to sign uploads' + return null + } + const api = checkUrl('the kubo API', s.api, 'local', { originOnly: true }) + if (api) return api + if (s.gateway !== '') { + const gw = checkUrl('the gateway', s.gateway, 'local', { originOnly: true }) + if (gw) return gw + if (new URL(s.gateway).hostname === 'localhost') { + return 'use http://127.0.0.1: for a local gateway: kubo redirects localhost to .ipfs.localhost subdomains, which the browser blocks' + } + } + if (s.publicGateway !== '') { + const pg = checkUrl('the public gateway', s.publicGateway, 'published', { originOnly: true }) + if (pg) return pg + } + if (s.kind === 'ipfs-pinning-service') { + const pe = checkUrl('the pinning endpoint', s.pinningEndpoint, 'local') + if (pe) return pe + if (!p.secrets.pinningToken) return 'the pinning service needs an access token' + } + return null +} + +const BUCKET = /^[a-z0-9._-]{1,63}$/ + +/** + * Why this profile's published addresses (the S3 public URL, the IPFS public gateway) cannot + * be recorded on chain, or null. They are what every reader fetches, so they must be public + * https: an address on this machine or a private network works only for its uploader, and would + * point other readers' browsers at their own local services. + */ +export function publishProblem(p: StorageProfile): string | null { + const s = p.settings + const published = s.kind === 's3' ? s.publicUrl : s.kind === 'platform' ? '' : s.publicGateway + if (published === '' || isPublicHttpsUrl(published)) return null + let host = published + try { + host = new URL(published).host + } catch { + /* keep the raw value */ + } + return `${host} is only reachable from this machine or its network, so other people cannot read what is stored there. The public address is recorded on chain for everyone: use a public https URL (a bucket domain, a CDN, or a tunnel).` +} + +/** The key prefix normalized: '' or ending in exactly one `/` (parity with forge-core). */ +export function normalizedPrefix(prefix: string): string { + const p = prefix.replace(/^\/+|\/+$/g, '') + return p === '' ? '' : `${p}/` +} + +/** The content-addressed object key of an artifact (`packs/.pack`, as the CLI writes). */ +export function artifactKey(settings: Extract, sha256Hex: string): string { + return `${normalizedPrefix(settings.prefix)}packs/${sha256Hex}.pack` +} + +// --------------------------------------------------------------------------- +// Persisted shapes (zod: what comes back from IndexedDB is parsed, not cast) +// --------------------------------------------------------------------------- + +const httpString = z.string().max(300) + +const s3Schema = z + .object({ + kind: z.literal('s3'), + provider: z.enum(['r2', 'b2', 'aws', 'minio']), + endpoint: httpString, + region: z.string().regex(/^[a-z0-9-]{1,64}$/), + bucket: z.string().regex(BUCKET), + pathStyle: z.boolean(), + publicUrl: httpString, + prefix: z.string().max(200), + }) + .strict() + +const ipfsSchema = z + .object({ + kind: z.enum(['ipfs-kubo', 'ipfs-pinning-service']), + provider: z.enum(['kubo', 'pinning']), + api: httpString, + gateway: httpString, + publicGateway: httpString, + pinningEndpoint: httpString, + }) + .strict() + // The kind and the tile must agree: a kubo tile is never a pinning profile, and back. + .refine((v) => (v.kind === 'ipfs-kubo') === (v.provider === 'kubo'), 'the IPFS kind and provider disagree') + +const platformSchema = z.object({ kind: z.literal('platform'), provider: z.literal('platform') }).strict() + +export const profilePublicSchema = z.union([s3Schema, ipfsSchema, platformSchema]) + +const headerValue = (max: number) => z.string().max(max).refine(isHeaderSafe).optional() + +export const profileSecretsSchema = z + .object({ + accessKeyId: headerValue(256), + secretAccessKey: z.string().max(512).optional(), + sessionToken: headerValue(4096), + apiAuth: headerValue(1024), + pinningToken: headerValue(4096), + }) + .strict() + +/** A repo's (or the default) browser-push storage policy. */ +export interface StoragePolicy { + /** Profile names, in order. Empty: no storage configured. */ + readonly targets: readonly string[] + /** Copies that must confirm (1..targets). */ + readonly replicas: number + /** Store on Platform when the targets cannot confirm `replicas` (asks, with the price, first). */ + readonly platformFallback: boolean +} + +export const policySchema = z.object({ + targets: z.array(z.string().max(64)).max(8), + replicas: z.number().int().min(1).max(8), + platformFallback: z.boolean(), +}) + +/** The replication choices of the wizard (§3.1 step 4). */ +export type ReplicationChoice = 'one' | 'all' | 'fallback' + +/** A policy for `targets` under a replication choice. */ +export function policyFor(targets: readonly string[], choice: ReplicationChoice): StoragePolicy { + return { + targets: [...targets], + replicas: choice === 'all' ? Math.max(1, targets.length) : 1, + platformFallback: choice === 'fallback', + } +} + +/** `policy` with the profile `from` renamed to `to` (a renamed profile keeps its place). */ +export function renameInPolicy(policy: StoragePolicy, from: string, to: string): StoragePolicy { + return { ...policy, targets: policy.targets.map((t) => (t === from ? to : t)) } +} + +/** Which replication choice a policy corresponds to (for the radio). */ +export function choiceOf(policy: StoragePolicy): ReplicationChoice { + if (policy.platformFallback) return 'fallback' + return policy.replicas > 1 && policy.replicas >= policy.targets.length ? 'all' : 'one' +} diff --git a/forge-web/lib/storage/s3.ts b/forge-web/lib/storage/s3.ts new file mode 100644 index 000000000..6e3370380 --- /dev/null +++ b/forge-web/lib/storage/s3.ts @@ -0,0 +1,178 @@ +/** + * A minimal S3 client for the browser: signed PUT / GET / HEAD / DELETE of one object, and + * anonymous reads through the bucket's public URL. Port of the request shapes of forge-core + * `backends/s3.rs` (path-style or virtual-hosted URLs, the key percent-encoded once, the body + * hash in `x-amz-content-sha256` so the store verifies the upload server-side). + * + * Browser specifics: `fetch` sets `Host` from the URL, so the signed host must be exactly the + * URL's; `redirect: 'error'` so a signed request is never replayed to another host; and every + * header the request carries beyond the CORS-safelisted ones (authorization, x-amz-*) needs the + * bucket's CORS rules to allow it, which the wizard tests before a push depends on it. + */ + +import { EMPTY_PAYLOAD_SHA256, amzDate, keyHasBadSegment, sha256Hex, signRequest, uriEncode } from './sigv4' +import type { ProfilePublic, ProfileSecrets } from './profiles' +import { TimeoutError, isHeaderSafe, timedFetch, type TimedResponse } from './util' + +export type S3Settings = Extract + +/** An S3 request that failed: the operation, the HTTP status (0: no response) and why. */ +export class S3Error extends Error { + constructor( + readonly op: string, + readonly status: number, + message: string, + ) { + super(message) + this.name = 'S3Error' + } +} + +/** The API URL of `key` (path-style: `endpoint/bucket/key`; else `bucket.host/key`). */ +export function objectUrl(s: S3Settings, key: string): URL { + if (keyHasBadSegment(key)) throw new S3Error('url', 0, `object key ${JSON.stringify(key)} has an empty, '.' or '..' segment`) + const endpoint = new URL(s.endpoint) + const encoded = uriEncode(key, true) + const url = s.pathStyle + ? new URL(`${endpoint.origin}/${s.bucket}/${encoded}`) + : new URL(`${endpoint.protocol}//${s.bucket}.${endpoint.host}/${encoded}`) + // What is signed must be what is sent: the parser must keep our encoding verbatim. + const intended = s.pathStyle ? `/${s.bucket}/${encoded}` : `/${encoded}` + if (url.pathname !== intended) throw new S3Error('url', 0, 'the request path would differ from the signed one; refusing to sign it') + return url +} + +/** The anonymous public URL of `key`. */ +export function publicObjectUrl(s: S3Settings, key: string): string { + return `${s.publicUrl.replace(/\/+$/, '')}/${uriEncode(key, true)}` +} + +/** The `s3://bucket/key` locator a manifest records next to the public URL (CLI read path). */ +export function s3Uri(s: S3Settings, key: string): string { + return `s3://${s.bucket}/${key}` +} + +async function send( + s: S3Settings, + secrets: ProfileSecrets, + op: string, + method: 'GET' | 'PUT' | 'HEAD' | 'DELETE', + key: string, + opts: { body?: Uint8Array; contentType?: string; range?: string; signal?: AbortSignal | undefined } = {}, +): Promise { + const url = objectUrl(s, key) + const payloadHash = opts.body ? await sha256Hex(opts.body) : EMPTY_PAYLOAD_SHA256 + const extra: [string, string][] = [] + if (opts.contentType) extra.push(['content-type', opts.contentType]) + if (opts.range) extra.push(['range', opts.range]) + if (!secrets.accessKeyId || !secrets.secretAccessKey) throw new S3Error(op, 0, 'no credentials: add the access key id and secret') + // A header value fetch refuses would be quoted in its error: refuse it here, without echoing it. + if (![secrets.accessKeyId, secrets.sessionToken ?? ''].every(isHeaderSafe)) { + throw new S3Error(op, 0, 'the access key id or session token holds characters an HTTP header cannot carry; paste it again') + } + const signed = await signRequest( + { + method, + host: url.host, + canonicalUri: url.pathname, + headers: extra, + payloadHash, + region: s.region, + service: 's3', + contentSha256Header: true, + }, + { + accessKeyId: secrets.accessKeyId, + secretAccessKey: secrets.secretAccessKey, + ...(secrets.sessionToken ? { sessionToken: secrets.sessionToken } : {}), + }, + amzDate(), + ) + const headers = new Headers([...extra, ...signed]) + try { + return await timedFetch( + url, + { + method, + headers, + ...(opts.body ? { body: new Uint8Array(opts.body) } : {}), + redirect: 'error', + credentials: 'omit', + cache: 'no-store', + }, + { uploadBytes: opts.body?.length ?? 0, signal: opts.signal }, + ) + } catch (e) { + if (e instanceof TimeoutError) throw new S3Error(op, 0, `${op} ${url.host}: ${e.message}`) + // A CORS refusal, a redirect (refused: a signed request is never replayed elsewhere) and a + // network failure all look the same to a page; the probe tells them apart. + throw new S3Error(op, 0, `${op} ${url.host}: the request was blocked, redirected or the host is unreachable`) + } +} + +async function failed(op: string, r: TimedResponse): Promise { + const resp = r.resp + let code = '' + try { + const text = (await r.text()).slice(0, 2000) + code = /([^<]{1,64})<\/Code>/.exec(text)?.[1] ?? '' + } catch { + /* body unreadable */ + } + return new S3Error(op, resp.status, `${op} failed: HTTP ${resp.status}${code ? ` ${code}` : ''}${statusHint(resp.status)}`) +} + +/** What an S3 error status usually means, as a suffix for the message ('' when nothing useful). */ +function statusHint(status: number): string { + switch (status) { + case 403: + return ' — check the key, its bucket permissions, the region (R2 wants auto) and the clock' + case 404: + return ' — check the bucket name and endpoint' + case 301: + case 307: + return ' — the bucket lives in another region; set that endpoint' + default: + return '' + } +} + +/** + * An anonymous GET of `key` through the bucket's public URL (what every reader uses), whole or + * a `Range`: the check that a copy is readable by anyone, not just by the credential holder. + */ +export async function getPublic(s: S3Settings, key: string, range?: string): Promise<{ status: number; bytes: Uint8Array; contentRange: string | null }> { + const r = await timedFetch(publicObjectUrl(s, key), { credentials: 'omit', cache: 'no-store', ...(range ? { headers: { Range: range } } : {}) }) + return { status: r.resp.status, contentRange: r.resp.headers.get('content-range'), bytes: await r.bytes() } +} + +/** Signed PUT of `bytes` at `key`. */ +export async function putObject(s: S3Settings, secrets: ProfileSecrets, key: string, bytes: Uint8Array, contentType = 'application/octet-stream', signal?: AbortSignal): Promise { + const r = await send(s, secrets, 'signed PUT', 'PUT', key, { body: bytes, contentType, signal }) + if (!r.resp.ok) throw await failed('signed PUT', r) + await r.bytes() +} + +/** Signed GET of `key` (whole object, or a `Range`). */ +export async function getObject(s: S3Settings, secrets: ProfileSecrets, key: string, range?: string, signal?: AbortSignal): Promise { + const r = await send(s, secrets, 'signed GET', 'GET', key, { range, signal }) + if (!r.resp.ok) throw await failed('signed GET', r) + return r.bytes() +} + +/** Signed HEAD: the object's size, or null when it does not exist. */ +export async function headObject(s: S3Settings, secrets: ProfileSecrets, key: string): Promise { + const r = await send(s, secrets, 'signed HEAD', 'HEAD', key) + if (!r.resp.ok && r.resp.status !== 404) throw await failed('signed HEAD', r) + r.discard() + if (r.resp.status === 404) return null + const len = Number(r.resp.headers.get('content-length')) + return Number.isFinite(len) ? len : 0 +} + +/** Signed DELETE of `key` (idempotent: a missing object is fine). */ +export async function deleteObject(s: S3Settings, secrets: ProfileSecrets, key: string): Promise { + const r = await send(s, secrets, 'delete', 'DELETE', key) + if (!r.resp.ok && r.resp.status !== 404) throw await failed('delete', r) + r.discard() +} diff --git a/forge-web/lib/storage/sigv4.test.ts b/forge-web/lib/storage/sigv4.test.ts new file mode 100644 index 000000000..17824916f --- /dev/null +++ b/forge-web/lib/storage/sigv4.test.ts @@ -0,0 +1,160 @@ +/** + * SigV4 parity with forge-core `backends/sigv4.rs`: the same AWS test-suite vectors and S3 + * API reference examples, so the browser and the CLI sign byte for byte alike. + */ + +import { describe, expect, it } from 'vitest' + +import { + EMPTY_PAYLOAD_SHA256, + amzDate, + canonicalHeaders, + canonicalQuery, + keyHasBadSegment, + sha256Hex, + signRequest, + uriEncode, + type SigningKeys, +} from './sigv4' + +const SUITE: SigningKeys = { accessKeyId: 'AKIDEXAMPLE', secretAccessKey: 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY' } +const SUITE_DATE = { date: '20150830', datetime: '20150830T123600Z' } + +async function suiteSign( + method: string, + path: string, + query: [string, string][] = [], + headers: [string, string][] = [], + sessionToken?: string, +): Promise { + const out = await signRequest( + { + method, + host: 'example.amazonaws.com', + canonicalUri: uriEncode(path, true), + query, + headers, + payloadHash: EMPTY_PAYLOAD_SHA256, + region: 'us-east-1', + service: 'service', + contentSha256Header: false, + }, + sessionToken ? { ...SUITE, sessionToken } : SUITE, + SUITE_DATE, + ) + const auth = out.find(([k]) => k === 'authorization')?.[1] ?? '' + return auth.split('Signature=')[1] ?? '' +} + +describe('SigV4 test suite (generic service)', () => { + it.each([ + ['get-vanilla', 'GET', '/', [], [], '5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31'], + ['get-space-unnormalized', 'GET', '/example space/', [], [], '652487583200325589f1fba4c7e578f72c47cb61beeca81406b39ddec1366741'], + ['get-utf8', 'GET', '/ሴ', [], [], '8318018e0b0f223aa2bbf98705b62bb787dc9c0e678f255a891fd03141be5d85'], + [ + 'get-unreserved', + 'GET', + '/-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz', + [], + [], + '07ef7494c76fa4850883e2b006601f940f8a34d404d0cfa977f52a65bbf5f24f', + ], + ['query-order-key-case', 'GET', '/', [['Param2', 'value2'], ['Param1', 'value1']], [], 'b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500'], + ['utf8-query-key', 'GET', '/', [['ሴ', 'bar']], [], '2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04'], + ['post-header-key-sort', 'POST', '/', [], [['My-Header1', 'value1']], 'c5410059b04c1ee005303aed430f6e6645f61f4dc9e1461ec8f8916fdf18852c'], + ] as const)('%s', async (_name, method, path, query, headers, want) => { + expect(await suiteSign(method, path, [...query] as [string, string][], [...headers] as [string, string][])).toBe(want) + }) + + it('signs the session token', async () => { + expect(await suiteSign('GET', '/', [], [], '6e86291e8372ff2a2260956d9b8aae1d763fbf315fa00fa31553b73ebf194267')).toBe( + '07ec1639c89043aa0e3e2de82b96708f198cceab042d4a97044c66dd9f74e7f8', + ) + }) +}) + +describe('S3 API reference examples', () => { + const S3: SigningKeys = { accessKeyId: 'AKIAIOSFODNN7EXAMPLE', secretAccessKey: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' } + const DATE = amzDate(Date.UTC(2013, 4, 24)) + + it('formats the UTC stamp', () => { + expect(DATE).toEqual({ date: '20130524', datetime: '20130524T000000Z' }) + expect(amzDate(1_440_938_160_000).datetime).toBe('20150830T123600Z') + expect(amzDate(951_782_400_000).date).toBe('20000229') + }) + + it('GET object with a range', async () => { + const out = await signRequest( + { + method: 'GET', + host: 'examplebucket.s3.amazonaws.com', + canonicalUri: '/test.txt', + headers: [['range', 'bytes=0-9']], + payloadHash: EMPTY_PAYLOAD_SHA256, + region: 'us-east-1', + service: 's3', + contentSha256Header: true, + }, + S3, + DATE, + ) + expect(out.find(([k]) => k === 'authorization')?.[1]).toBe( + 'AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, ' + + 'SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, ' + + 'Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41', + ) + }) + + it('PUT object whose key needs encoding', async () => { + const payload = await sha256Hex(new TextEncoder().encode('Welcome to Amazon S3.')) + expect(payload).toBe('44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072') + const uri = `/${uriEncode('test$file.text', true)}` + expect(uri).toBe('/test%24file.text') + const out = await signRequest( + { + method: 'PUT', + host: 'examplebucket.s3.amazonaws.com', + canonicalUri: uri, + headers: [ + ['Date', 'Fri, 24 May 2013 00:00:00 GMT'], + ['x-amz-storage-class', 'REDUCED_REDUNDANCY'], + ], + payloadHash: payload, + region: 'us-east-1', + service: 's3', + contentSha256Header: true, + }, + S3, + DATE, + ) + expect(out.find(([k]) => k === 'authorization')?.[1]).toMatch( + /SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd$/, + ) + }) +}) + +describe('canonicalization helpers', () => { + it('encodes like forge-core', () => { + expect(uriEncode('a b+c=d&e', true)).toBe('a%20b%2Bc%3Dd%26e') + expect(uriEncode('dir/file', true)).toBe('dir/file') + expect(uriEncode('dir/file', false)).toBe('dir%2Ffile') + expect(uriEncode('ü', true)).toBe('%C3%BC') + expect(uriEncode('100%', true)).toBe('100%25') + }) + + it('trims, collapses and merges headers', () => { + expect(canonicalHeaders([['X-B', ' two words '], ['x-a', '1'], ['X-A', '2']])).toEqual({ + canonical: 'x-a:1,2\nx-b:two words\n', + signed: 'x-a;x-b', + }) + }) + + it('keeps = on an empty query value', () => { + expect(canonicalQuery([['b', ''], ['a', 'x y']])).toBe('a=x%20y&b=') + }) + + it('refuses keys an HTTP stack would normalize', () => { + expect(keyHasBadSegment('packs/abc.pack')).toBe(false) + for (const k of ['', 'a//b', './a', 'a/../b', 'a/']) expect(keyHasBadSegment(k)).toBe(true) + }) +}) diff --git a/forge-web/lib/storage/sigv4.ts b/forge-web/lib/storage/sigv4.ts new file mode 100644 index 000000000..43f0d462f --- /dev/null +++ b/forge-web/lib/storage/sigv4.ts @@ -0,0 +1,164 @@ +/** + * AWS Signature Version 4, for S3-compatible buckets (R2, B2, AWS S3, MinIO), on WebCrypto. + * + * A port of forge-core `backends/sigv4.rs` and pinned by the same vectors: the AWS SigV4 test + * suite and the worked examples in the S3 API reference (`sigv4.test.ts`). S3 canonicalization + * differs from generic SigV4 in one way that matters: the object path is percent-encoded ONCE, + * keeping `/`, and never normalized. A key with an empty, `.` or `..` segment is refused by the + * caller, because `fetch` would normalize it after signing and address another object. + * + * The secret access key is imported into a non-extractable HMAC `CryptoKey` for the first step + * of the key derivation and never formatted, logged or returned. + */ + +/** SHA-256 of the empty string: the payload hash of a body-less request. */ +export const EMPTY_PAYLOAD_SHA256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' + +const ALGORITHM = 'AWS4-HMAC-SHA256' +const enc = new TextEncoder() + +/** The credentials a request is signed with. */ +export interface SigningKeys { + readonly accessKeyId: string + readonly secretAccessKey: string + readonly sessionToken?: string +} + +/** A signing timestamp in the two forms SigV4 uses. */ +export interface AmzDate { + /** `YYYYMMDD` (the credential-scope date). */ + readonly date: string + /** `YYYYMMDDTHHMMSSZ` (the `x-amz-date` value). */ + readonly datetime: string +} + +/** The UTC stamp of `ms` since the epoch. */ +export function amzDate(ms: number = Date.now()): AmzDate { + const iso = new Date(ms).toISOString() // 2013-05-24T00:00:00.000Z + const datetime = `${iso.slice(0, 4)}${iso.slice(5, 7)}${iso.slice(8, 10)}T${iso.slice(11, 13)}${iso.slice(14, 16)}${iso.slice(17, 19)}Z` + return { date: datetime.slice(0, 8), datetime } +} + +/** + * RFC 3986 percent-encoding as SigV4 specifies it: every byte except `A-Z a-z 0-9 - . _ ~` + * becomes `%XX` (uppercase hex). With `keepSlash`, `/` stays (object-key paths). + */ +export function uriEncode(input: string, keepSlash: boolean): string { + let out = '' + for (const b of enc.encode(input)) { + const c = String.fromCharCode(b) + if (/[A-Za-z0-9\-._~]/.test(c) || (keepSlash && c === '/')) out += c + else out += `%${b.toString(16).toUpperCase().padStart(2, '0')}` + } + return out +} + +/** Whether an object key has an empty, `.` or `..` segment (see the module doc). */ +export function keyHasBadSegment(key: string): boolean { + return key === '' || key.split('/').some((s) => s === '' || s === '.' || s === '..') +} + +function hex(bytes: ArrayBuffer | Uint8Array): string { + const u = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes) + let s = '' + for (const b of u) s += b.toString(16).padStart(2, '0') + return s +} + +function buf(b: Uint8Array): Uint8Array { + return new Uint8Array(b) +} + +/** Lowercase-hex SHA-256 of `bytes`. */ +export async function sha256Hex(bytes: Uint8Array): Promise { + return hex(await crypto.subtle.digest('SHA-256', buf(bytes))) +} + +async function hmac(key: Uint8Array | CryptoKey, data: string): Promise { + const k = + key instanceof Uint8Array + ? await crypto.subtle.importKey('raw', buf(key), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']) + : key + return new Uint8Array(await crypto.subtle.sign('HMAC', k, enc.encode(data))) +} + +/** The derived signing key: `HMAC(HMAC(HMAC(HMAC("AWS4"+secret, date), region), service), "aws4_request")`. */ +async function signingKey(secret: string, date: string, region: string, service: string): Promise { + const root = await crypto.subtle.importKey('raw', enc.encode(`AWS4${secret}`), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']) + const kDate = await hmac(root, date) + const kRegion = await hmac(kDate, region) + const kService = await hmac(kRegion, service) + return hmac(kService, 'aws4_request') +} + +/** + * The canonical query: each key and value encoded (slashes too), sorted by encoded key then + * value, joined with `&`. An empty value keeps its `=`. + */ +export function canonicalQuery(pairs: readonly (readonly [string, string])[]): string { + return pairs + .map(([k, v]) => [uriEncode(k, false), uriEncode(v, false)] as const) + .sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0)) + .map(([k, v]) => `${k}=${v}`) + .join('&') +} + +/** + * Canonical headers and the `SignedHeaders` list: names lowercased, values trimmed with inner + * whitespace runs collapsed, sorted by name (stable), duplicates joined by `,`. + */ +export function canonicalHeaders(headers: readonly (readonly [string, string])[]): { canonical: string; signed: string } { + const norm = headers + .map(([k, v]) => [k.trim().toLowerCase(), v.split(/\s+/).filter((p) => p !== '').join(' ')] as const) + .map((h, i) => ({ h, i })) + .sort((a, b) => (a.h[0] < b.h[0] ? -1 : a.h[0] > b.h[0] ? 1 : a.i - b.i)) + .map((x) => x.h) + const merged: [string, string][] = [] + for (const [k, v] of norm) { + const last = merged[merged.length - 1] + if (last && last[0] === k) last[1] = `${last[1]},${v}` + else merged.push([k, v]) + } + return { + canonical: merged.map(([k, v]) => `${k}:${v}\n`).join(''), + signed: merged.map(([k]) => k).join(';'), + } +} + +/** One request to sign. */ +export interface RequestToSign { + readonly method: string + /** The `Host` exactly as sent: `host`, or `host:port` for a non-default port. */ + readonly host: string + /** The already-encoded path ({@link uriEncode} with `keepSlash`). */ + readonly canonicalUri: string + readonly query?: readonly (readonly [string, string])[] + /** Extra headers to sign (`range`, `content-type`); the signer adds its own. */ + readonly headers?: readonly (readonly [string, string])[] + /** Hex SHA-256 of the body, or {@link EMPTY_PAYLOAD_SHA256}. */ + readonly payloadHash: string + readonly region: string + readonly service: string + /** Send and sign `x-amz-content-sha256` (S3 requires it; the generic suite does not). */ + readonly contentSha256Header: boolean +} + +/** + * Sign `req`: the headers to add to the outgoing request, `authorization` last. `host` is + * signed but not returned; `fetch` derives it from the URL, which must produce `req.host`. + */ +export async function signRequest(req: RequestToSign, keys: SigningKeys, when: AmzDate): Promise<[string, string][]> { + const added: [string, string][] = [['x-amz-date', when.datetime]] + if (req.contentSha256Header) added.push(['x-amz-content-sha256', req.payloadHash]) + if (keys.sessionToken) added.push(['x-amz-security-token', keys.sessionToken]) + const all: [string, string][] = [['host', req.host], ...(req.headers ?? []).map(([k, v]) => [k, v] as [string, string]), ...added] + const { canonical, signed } = canonicalHeaders(all) + const creq = [req.method, req.canonicalUri, canonicalQuery(req.query ?? []), canonical, signed, req.payloadHash].join('\n') + const scope = `${when.date}/${req.region}/${req.service}/aws4_request` + const sts = [ALGORITHM, when.datetime, scope, await sha256Hex(enc.encode(creq))].join('\n') + const key = await signingKey(keys.secretAccessKey, when.date, req.region, req.service) + const signature = hex(await hmac(key, sts)) + key.fill(0) + added.push(['authorization', `${ALGORITHM} Credential=${keys.accessKeyId}/${scope}, SignedHeaders=${signed}, Signature=${signature}`]) + return added +} diff --git a/forge-web/lib/storage/storage.test.ts b/forge-web/lib/storage/storage.test.ts new file mode 100644 index 000000000..d0ea4d71d --- /dev/null +++ b/forge-web/lib/storage/storage.test.ts @@ -0,0 +1,434 @@ +/** + * The storage layer, offline: profile validation, policies, URI ordering and budgets, chunk + * splitting (parity with forge-core `pack::split`), the CORS fix blocks, the config sealed in + * the vault, and the replication engine against in-memory S3 and kubo fakes. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { resetMemoryStores, idbEntries, idbGet, idbPut } from '../idb' +import { lockVault, storeInVault, unlockWithPassphrase } from '../auth/vault' +import { splitChunks, manifestUrisProblem } from '../repo/push' +import { CHUNK_PAYLOAD_MAX, FIELD_MAX } from '../constants' +import { isPrivateHost, isPublicHttpsUrl } from '../net' +import { externalFetchUrls } from '../view/browse-source' +import { corsFix } from './cors' +import { cidV1RawLeaves } from './cid' +import { + PLATFORM_PROFILE, + artifactKey, + choiceOf, + policyFor, + profileProblem, + profileSecretsSchema, + publishProblem, + type ProfilePublic, + type StorageProfile, +} from './profiles' +import { + EMPTY_STORAGE_CONFIG, + discardStorageConfig, + loadStorageConfig, + policyForRepo, + policyProblem, + saveStorageConfig, + withProfile, + withRenamedProfile, + withRepoPolicy, + withoutProfile, +} from './store' +import { ReplicationError, PlatformDeclinedError, fitManifestUris, orderUris, storeArtifact } from './upload' +import { sha256Hex } from './sigv4' +import type { WriteAuth } from '../sdk' +import type { V2RepoRef } from '../repo/contract' +import type { EvoSDK } from '@dashevo/evo-sdk' + +const ID = '9r27eDsuXEqoMNymW1A2MKFrpBhzSkepVKwXrGzq9dUD' +const WIF = 'cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy' + +const S3: StorageProfile = { + name: 'r2-main', + settings: { + kind: 's3', + provider: 'r2', + endpoint: 'https://acct.r2.cloudflarestorage.com', + region: 'auto', + bucket: 'forge-byo', + pathStyle: true, + publicUrl: 'https://pub-9a1.r2.dev', + prefix: 'web', + }, + secrets: { accessKeyId: 'AKIDEXAMPLE', secretAccessKey: 'r2-secret-value' }, +} + +const KUBO: StorageProfile = { + name: 'kubo', + settings: { kind: 'ipfs-kubo', provider: 'kubo', api: 'https://kubo.example', gateway: 'https://kubo-gw.example', publicGateway: '', pinningEndpoint: '' }, + secrets: {}, +} + +const s3Of = (p: StorageProfile): Extract => p.settings as Extract + +describe('profiles', () => { + it('accepts a sound S3 profile and names what is wrong otherwise', () => { + expect(profileProblem(S3)).toBeNull() + const s3 = s3Of(S3) + const bad = (patch: Partial, secrets = S3.secrets): string | null => profileProblem({ ...S3, settings: { ...s3, ...patch }, secrets }) + expect(bad({ endpoint: 'https://x.example/bucket' })).toMatch(/origin/) + expect(bad({ endpoint: 'https://user:pw@x.example' })).toMatch(/user name/) + expect(bad({ endpoint: 'http://minio.example.org' })).toMatch(/https/) + // The API endpoint may be plain http on this machine (the user's own MinIO); what is + // published may not (the public URL, checked by publishProblem). + expect(bad({ endpoint: 'http://127.0.0.1:9000' })).toBeNull() + expect(bad({ endpoint: 'http://192.168.1.4:9000' })).toMatch(/https/) + expect(bad({ bucket: 'Forge' })).toMatch(/lowercase/) + expect(bad({ prefix: 'a/../b' })).toMatch(/segment/) + expect(bad({ publicUrl: '' })).toMatch(/public URL/) + expect(bad({ publicUrl: 'http://pub.example' })).toMatch(/recorded on chain/) + expect(bad({ pathStyle: false, endpoint: 'https://1.2.3.4' })).toMatch(/path-style/) + expect(bad({}, {})).toMatch(/access key/) + expect(profileProblem({ ...S3, name: 'a,b' })).toMatch(/name/) + }) + + it('reserves the Platform profile name', () => { + expect(profileProblem({ ...S3, name: PLATFORM_PROFILE })).toMatch(/reserved/) + expect(profileProblem({ name: PLATFORM_PROFILE, settings: { kind: 'platform', provider: 'platform' }, secrets: {} })).toBeNull() + }) + + it('refuses secrets an HTTP header cannot carry, without echoing them', () => { + const problem = profileProblem({ ...S3, secrets: { ...S3.secrets, sessionToken: 'tok\nsecret-part' } }) + expect(problem).toMatch(/session token/i) + expect(problem).not.toContain('secret-part') + expect(profileSecretsSchema.safeParse({ accessKeyId: 'ключ' }).success).toBe(false) + expect(profileSecretsSchema.safeParse({ accessKeyId: 'AKID', extra: 'x' }).success).toBe(false) + }) + + it('refuses a published address only its uploader can reach', () => { + expect(publishProblem(S3)).toBeNull() + for (const url of ['http://127.0.0.1:9000/forge-byo', 'https://192.168.1.4/b', 'https://10.0.0.8', 'https://minio.local/b', 'https://[::1]/b', 'https://[fd00::1]/b']) { + expect(publishProblem({ ...S3, settings: { ...s3Of(S3), publicUrl: url } }), url).toMatch(/other people cannot read/) + } + expect(publishProblem({ ...KUBO, settings: { ...KUBO.settings, publicGateway: 'http://localhost:8080' } as ProfilePublic })).toMatch(/other people/) + }) + + it('keys artifacts like the CLI', () => { + const s3 = s3Of(S3) + expect(artifactKey(s3, 'ab'.repeat(32))).toBe(`web/packs/${'ab'.repeat(32)}.pack`) + expect(artifactKey({ ...s3, prefix: '' }, 'cd')).toBe('packs/cd.pack') + expect(artifactKey({ ...s3, prefix: '/a/b//' }, 'cd')).toBe('a/b/packs/cd.pack') + }) + + it('maps replication choices to policies and back', () => { + expect(policyFor(['a', 'b'], 'one')).toEqual({ targets: ['a', 'b'], replicas: 1, platformFallback: false }) + expect(policyFor(['a', 'b'], 'all')).toEqual({ targets: ['a', 'b'], replicas: 2, platformFallback: false }) + expect(policyFor(['a'], 'fallback')).toEqual({ targets: ['a'], replicas: 1, platformFallback: true }) + for (const c of ['one', 'all', 'fallback'] as const) expect(choiceOf(policyFor(['a', 'b'], c))).toBe(c) + }) +}) + +describe('what readers may fetch', () => { + it('only public https hosts', () => { + for (const h of ['127.0.0.1', 'localhost', 'x.localhost', '10.1.2.3', '172.20.0.1', '192.168.0.9', '169.254.1.1', '100.64.0.1', '::1', 'fe80::1', 'fc00::5', '::ffff:127.0.0.1']) { + expect(isPrivateHost(h), h).toBe(true) + } + for (const h of ['pub-9a1.r2.dev', '8.8.8.8', '172.32.0.1', 'ipfs.io']) expect(isPrivateHost(h), h).toBe(false) + expect(isPublicHttpsUrl('http://pub.example/p')).toBe(false) + expect(isPublicHttpsUrl('https://pub.example/p')).toBe(true) + }) + + it('never makes a reader request a loopback or private URL a manifest records', () => { + expect( + externalFetchUrls(['http://127.0.0.1:9000/p', 'https://192.168.1.2/p', 'http://pub.example/p', 'https://pub.example/p', 'ipfs://bafyok', 'ipfs://bafy/../api'], ['https://ipfs.io']), + ).toEqual(['https://pub.example/p', 'https://ipfs.io/ipfs/bafyok']) + }) +}) + +describe('the stored configuration', () => { + it('prunes a removed profile out of every policy', () => { + let c = withProfile(withProfile(EMPTY_STORAGE_CONFIG, S3), KUBO) + c = { ...c, defaultPolicy: policyFor(['r2-main', 'kubo'], 'all') } + c = withRepoPolicy(c, 'R1', policyFor(['kubo'], 'one')) + const after = withoutProfile(c, 'kubo') + expect(after.profiles.map((p) => p.name)).toEqual(['r2-main']) + expect(after.defaultPolicy).toEqual({ targets: ['r2-main'], replicas: 1, platformFallback: false }) + expect(after.repoPolicies).toEqual({}) + expect(policyForRepo(after, 'R1')).toEqual(after.defaultPolicy) + }) + + it('renames a profile inside every policy', () => { + let c = withProfile(withProfile(EMPTY_STORAGE_CONFIG, S3), KUBO) + c = { ...c, defaultPolicy: policyFor(['r2-main', 'kubo'], 'all'), lastTests: { kubo: { at: 1, ok: true } } } + c = withRepoPolicy(c, 'R1', policyFor(['kubo'], 'one')) + const after = withRenamedProfile(c, 'kubo', { ...KUBO, name: 'my-node' }) + expect(after.profiles.map((p) => p.name)).toEqual(['my-node', 'r2-main']) + expect(after.defaultPolicy?.targets).toEqual(['r2-main', 'my-node']) + expect(after.repoPolicies['R1']?.targets).toEqual(['my-node']) + expect(after.lastTests).toEqual({}) + }) + + it('refuses a policy naming unknown profiles or too many copies', () => { + const c = withProfile(EMPTY_STORAGE_CONFIG, S3) + expect(policyProblem(c, policyFor(['r2-main'], 'one'))).toBeNull() + expect(policyProblem(c, policyFor(['nope'], 'one'))).toMatch(/no storage profile/) + expect(policyProblem(c, { targets: ['r2-main'], replicas: 2, platformFallback: false })).toMatch(/copies/) + expect(policyProblem(c, { targets: [], replicas: 1, platformFallback: false })).toMatch(/at least one/) + }) +}) + +describe('the configuration sealed in the vault', () => { + beforeEach(() => { + resetMemoryStores() + lockVault() + }) + + it('stores secrets encrypted, reads them back unlocked, and refuses when locked', async () => { + await storeInVault('devnet', { identityId: ID, keyId: 5, wif: WIF }, { passphrase: 'correct horse battery' }) + const config = withRepoPolicy(withProfile(EMPTY_STORAGE_CONFIG, S3), 'R1', policyFor(['r2-main'], 'one')) + await saveStorageConfig('devnet', ID, config) + const dump = JSON.stringify(await idbEntries('vault'), (_k, v: unknown) => (v instanceof Uint8Array ? Array.from(v) : v)) + expect(dump).not.toContain('r2-secret-value') + expect(dump).not.toContain('forge-byo') + expect(await loadStorageConfig('devnet', ID)).toEqual(config) + lockVault() + await expect(loadStorageConfig('devnet', ID)).rejects.toThrow(/unlock/) + await unlockWithPassphrase('devnet', ID, 'correct horse battery') + expect((await loadStorageConfig('devnet', ID)).profiles[0]?.secrets.secretAccessKey).toBe('r2-secret-value') + }, 60_000) + + it('carries the storage settings across a key renewal made while unlocked', async () => { + await storeInVault('devnet', { identityId: ID, keyId: 5, wif: WIF }, { passphrase: 'correct horse battery' }) + await saveStorageConfig('devnet', ID, withProfile(EMPTY_STORAGE_CONFIG, S3)) + const outcome = await storeInVault('devnet', { identityId: ID, keyId: 6, wif: WIF }, { passphrase: 'a different passphrase' }) + expect(outcome.storageSettingsDropped).toBe(false) + lockVault() + await unlockWithPassphrase('devnet', ID, 'a different passphrase') + expect((await loadStorageConfig('devnet', ID)).profiles.map((p) => p.name)).toEqual(['r2-main']) + }, 90_000) + + it('says so when a renewal made while locked cannot carry them', async () => { + await storeInVault('devnet', { identityId: ID, keyId: 5, wif: WIF }, { passphrase: 'correct horse battery' }) + await saveStorageConfig('devnet', ID, withProfile(EMPTY_STORAGE_CONFIG, S3)) + lockVault() + const outcome = await storeInVault('devnet', { identityId: ID, keyId: 6, wif: WIF }, { passphrase: 'a different passphrase' }) + expect(outcome.storageSettingsDropped).toBe(true) + expect(await loadStorageConfig('devnet', ID)).toEqual(EMPTY_STORAGE_CONFIG) + }, 90_000) + + it('can discard settings sealed under a key it no longer has', async () => { + await storeInVault('devnet', { identityId: ID, keyId: 5, wif: WIF }, { passphrase: 'correct horse battery' }) + await saveStorageConfig('devnet', ID, withProfile(EMPTY_STORAGE_CONFIG, S3)) + const stale = await idbGet('vault', `vault-storage:devnet:${ID}`) + await storeInVault('devnet', { identityId: ID, keyId: 6, wif: WIF }, { passphrase: 'a different passphrase' }) + await idbPut('vault', `vault-storage:devnet:${ID}`, stale) + await expect(loadStorageConfig('devnet', ID)).rejects.toThrow(/do not open/) + await discardStorageConfig('devnet', ID) + expect(await loadStorageConfig('devnet', ID)).toEqual(EMPTY_STORAGE_CONFIG) + }, 90_000) +}) + +describe('chunks and manifests', () => { + it('splits exactly as forge-core pack::split', () => { + expect(splitChunks(new Uint8Array(0))).toEqual([]) + const bytes = new Uint8Array(CHUNK_PAYLOAD_MAX * 2 + FIELD_MAX + 7).map((_, i) => i % 251) + const chunks = splitChunks(bytes) + expect(chunks.map((c) => c.seq)).toEqual([0, 1, 2]) + expect(chunks.map((c) => c.fields.map((f) => f.length))).toEqual([[FIELD_MAX, FIELD_MAX, FIELD_MAX], [FIELD_MAX, FIELD_MAX, FIELD_MAX], [FIELD_MAX, 7]]) + const joined = new Uint8Array(chunks.flatMap((c) => c.fields.flatMap((f) => Array.from(f)))) + expect(joined).toEqual(bytes) + }) + + it('orders URIs platform, then https, then the rest, and fits the budget', () => { + expect(orderUris([['https://a/p', 's3://b/p'], ['ipfs://bafy', 'https://gw/ipfs/bafy'], ['platform://C/R/O/h']])).toEqual([ + 'platform://C/R/O/h', + 'https://a/p', + 'https://gw/ipfs/bafy', + 's3://b/p', + 'ipfs://bafy', + ]) + const many = Array.from({ length: 5 }, (_, i) => [`https://h${i}.example/p`, `s3://b${i}/p`]) + expect(fitManifestUris(orderUris(many))).toHaveLength(5) + expect(() => fitManifestUris([`https://x.example/${'a'.repeat(400)}`])).toThrow(/do not fit/) + expect(manifestUrisProblem([])).toMatch(/no confirmed copy/) + }) + + it('never records an address others cannot read', () => { + expect(() => fitManifestUris(['http://127.0.0.1:9000/b/p', 's3://b/p'])).toThrow(/only public https/) + expect(() => fitManifestUris(['https://192.168.1.2/p'])).toThrow(/only public https/) + expect(() => fitManifestUris(['http://pub.example/p'])).toThrow(/only public https/) + }) +}) + +describe('CORS fix blocks', () => { + it('are valid JSON naming the bucket, Range for reads and PUT for this origin', () => { + for (const p of ['r2', 'aws', 'b2'] as const) { + const fix = corsFix(p, 'my-bucket', 'https://forge.dashhq.org') + expect(fix.where + fix.text).toContain('my-bucket') + const text = JSON.stringify(JSON.parse(fix.text) as unknown).toLowerCase() + expect(text).toContain('range') + expect(text).toMatch(/put/) + expect(text).toContain('https://forge.dashhq.org') + expect(text).toContain('x-amz-content-sha256') + } + }) + + it('keeps MinIO reads open to every origin', () => { + expect(corsFix('minio', 'b', 'https://forge.dashhq.org').text).toContain('cors_allow_origin="*"') + }) + + it('gives kubo a restricted token and merges the origin into the existing list', () => { + const { where, text } = corsFix('kubo', '', 'https://forge.dashhq.org') + expect(where).toMatch(/admin interface/) + expect(text).toContain('API.Authorizations.dash-forge') + for (const path of ['/api/v0/add', '/api/v0/pin/ls', '/api/v0/pin/rm', '/api/v0/id', '/api/v0/version']) expect(text).toContain(path) + expect(text).toMatch(/\(ipfs config API\.HTTPHeaders\.Access-Control-Allow-Origin 2>\/dev\/null \|\| echo null\) \| jq -c '\(\. \/\/ \[\]\) \+ \["https:\/\/forge\.dashhq\.org"\] \| unique'/) + const commands = text.split('\n').filter((l) => !l.startsWith('#')) + expect(commands.join('\n')).not.toMatch(/Access-Control-Allow-Origin '\["https:\/\/forge\.dashhq\.org"\]'/) + // Any Authorizations entry locks the RPC API for everyone else: an owner token comes first. + expect(text).toMatch(/API\.Authorizations\.owner .*\\"AllowedPaths\\": \[\\"\/api\/v0\\"\]/) + expect(text.indexOf('API.Authorizations.owner')).toBeLessThan(text.indexOf('API.Authorizations.dash-forge')) + }) +}) + +// --------------------------------------------------------------------------- +// The replication engine against in-memory S3 and kubo +// --------------------------------------------------------------------------- + +const REPO: V2RepoRef = { kind: 'v2', forge: { core: 'CORE', collab: 'COLLAB', group: 'G' }, repoId: 'REPO', ownerId: ID, name: 'r', visibility: 'public' } +const AUTH: WriteAuth = { identityId: ID, network: 'devnet', getSigningKeyWif: () => '' } +const SDK = {} as EvoSDK + +/** Serve `obj` for a (possibly ranged) GET. */ +function serve(obj: Uint8Array, init?: RequestInit): Response { + const range = new Headers(init?.headers).get('range') + const m = range ? /^bytes=(\d+)-(\d+)$/.exec(range) : null + if (!m) return new Response(new Uint8Array(obj), { status: 200 }) + const [start, end] = [Number(m[1]), Number(m[2])] + return new Response(obj.slice(start, end + 1), { status: 206, headers: { 'content-range': `bytes ${start}-${end}/${obj.length}` } }) +} + +function fakeNetwork(opts: { s3Down?: boolean; corruptOnce?: boolean; publicDenied?: boolean } = {}) { + const objects = new Map() + let corrupt = opts.corruptOnce === true + const puts: string[] = [] + const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = new URL(String(input)) + const method = init?.method ?? 'GET' + if (url.host === 'acct.r2.cloudflarestorage.com') { + if (opts.s3Down) throw new TypeError('Failed to fetch') + const key = url.pathname.replace(/^\/forge-byo\//, '') + if (method === 'PUT') { + const body = new Uint8Array(init?.body as ArrayBuffer | Uint8Array) + puts.push(key) + objects.set(key, new Uint8Array(corrupt ? body.map((b) => b ^ 1) : body)) + corrupt = false + return new Response(null, { status: 200 }) + } + const obj = objects.get(key) + if (!obj) return new Response(null, { status: 404 }) + if (method === 'HEAD') return new Response(null, { status: 200, headers: { 'content-length': String(obj.length) } }) + return serve(obj, init) + } + if (url.host === 'pub-9a1.r2.dev') { + if (opts.publicDenied) return new Response('AccessDenied', { status: 403 }) + const obj = objects.get(url.pathname.slice(1)) + return obj ? serve(obj, init) : new Response(null, { status: 404 }) + } + if (url.host === 'kubo.example') { + if (url.pathname.endsWith('/add')) { + const file = (init?.body as FormData).get('file') as Blob + const bytes = new Uint8Array(await file.arrayBuffer()) + const cid = cidV1RawLeaves(bytes) + objects.set(`ipfs:${cid}`, bytes) + return new Response(JSON.stringify({ Hash: cid })) + } + if (url.pathname.endsWith('/pin/ls')) { + const cid = url.searchParams.get('arg') ?? '' + return objects.has(`ipfs:${cid}`) ? new Response(JSON.stringify({ Keys: { [cid]: { Type: 'recursive' } } })) : new Response('{"Message":"not pinned"}', { status: 500 }) + } + return new Response('{}') + } + if (url.host === 'kubo-gw.example' || url.host === 'ipfs.example') { + const obj = objects.get(`ipfs:${url.pathname.split('/').pop() ?? ''}`) + return obj ? serve(obj, init) : new Response(null, { status: 404 }) + } + throw new Error(`unexpected ${url}`) + }) + return { fetchMock, objects, puts } +} + +describe('storeArtifact', () => { + const bytes = new TextEncoder().encode('PACK a small pack') + afterEach(() => vi.unstubAllGlobals()) + + it('stores on S3, verifies through the API and the public URL, and records the public URL first', async () => { + const net = fakeNetwork() + vi.stubGlobal('fetch', net.fetchMock) + const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main'], 'one'), profiles: [S3], confirmPlatform: async () => false }) + const hash = await sha256Hex(bytes) + expect(stored).toMatchObject({ packHash: hash, storage: 1, chunkCount: 0, confirmed: ['r2-main'], failures: [] }) + expect(stored.uris).toEqual([`https://pub-9a1.r2.dev/web/packs/${hash}.pack`, `s3://forge-byo/web/packs/${hash}.pack`]) + const put = net.fetchMock.mock.calls.find(([, init]) => init?.method === 'PUT') + expect(new Headers(put?.[1]?.headers).get('authorization')).toMatch(/^AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE\//) + expect(new Headers(put?.[1]?.headers).get('x-amz-content-sha256')).toBe(hash) + // The public URL was read anonymously (no Authorization) before the copy counted. + const pub = net.fetchMock.mock.calls.find(([u]) => String(u).startsWith('https://pub-9a1.r2.dev/')) + expect(pub).toBeDefined() + expect(new Headers(pub?.[1]?.headers).get('authorization')).toBeNull() + }) + + it('does not count a copy whose public URL others cannot read', async () => { + vi.stubGlobal('fetch', fakeNetwork({ publicDenied: true }).fetchMock) + await expect(storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main'], 'one'), profiles: [S3], confirmPlatform: async () => false })).rejects.toThrow( + /public URL \(pub-9a1\.r2\.dev\) answered HTTP 403/, + ) + }) + + it('refuses to upload to a profile whose public address only this machine can read', async () => { + const net = fakeNetwork() + vi.stubGlobal('fetch', net.fetchMock) + const local = { ...S3, settings: { ...s3Of(S3), publicUrl: 'https://192.168.1.20/forge-byo' } } + await expect(storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main'], 'one'), profiles: [local], confirmPlatform: async () => false })).rejects.toThrow(/other people cannot read/) + expect(net.puts).toEqual([]) + }) + + it('re-uploads once when the stored copy does not verify', async () => { + const net = fakeNetwork({ corruptOnce: true }) + vi.stubGlobal('fetch', net.fetchMock) + const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main'], 'one'), profiles: [S3], confirmPlatform: async () => false }) + expect(stored.confirmed).toEqual(['r2-main']) + expect(net.puts).toHaveLength(2) + }) + + it('fails the policy (writing nothing to Platform) when too few targets confirm', async () => { + vi.stubGlobal('fetch', fakeNetwork({ s3Down: true }).fetchMock) + const confirm = vi.fn(async () => true) + const run = storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main', 'kubo'], 'all'), profiles: [S3, KUBO], confirmPlatform: confirm }) + await expect(run).rejects.toBeInstanceOf(ReplicationError) + await expect(run).rejects.toThrow(/1 of 2 required .*r2-main: signed PUT.*Nothing was written to Platform/) + expect(confirm).not.toHaveBeenCalled() + }) + + it('asks, with the price, before a policy that names Platform writes any chunk', async () => { + vi.stubGlobal('fetch', fakeNetwork().fetchMock) + const platform: StorageProfile = { name: PLATFORM_PROFILE, settings: { kind: 'platform', provider: 'platform' }, secrets: {} } + const confirm = vi.fn(async () => false) + const run = storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor([PLATFORM_PROFILE], 'one'), profiles: [platform], confirmPlatform: confirm }) + await expect(run).rejects.toBeInstanceOf(ReplicationError) + await expect(run).rejects.toThrow(/declined.*Nothing was written to Platform/) + expect(confirm).toHaveBeenCalledWith(expect.objectContaining({ bytes: bytes.length, estimateCredits: expect.any(Number), reason: expect.stringMatching(/includes Dash Platform/) })) + }) + + it('verifies the kubo CID, the node gateway and the public gateway', async () => { + vi.stubGlobal('fetch', fakeNetwork().fetchMock) + const withPublic = { ...KUBO, settings: { ...KUBO.settings, publicGateway: 'https://ipfs.example' } as ProfilePublic } + const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['kubo'], 'one'), profiles: [withPublic], confirmPlatform: async () => false }) + const cid = cidV1RawLeaves(bytes) + expect(stored.uris).toEqual([`https://ipfs.example/ipfs/${cid}`, `ipfs://${cid}`]) + }) + + it('asks before storing on Platform when nothing is configured, and stops when declined', async () => { + const confirm = vi.fn(async () => false) + await expect(storeArtifact(SDK, AUTH, REPO, bytes, { policy: null, profiles: [], confirmPlatform: confirm })).rejects.toBeInstanceOf(PlatformDeclinedError) + expect(confirm).toHaveBeenCalledWith(expect.objectContaining({ bytes: bytes.length, estimateCredits: expect.any(Number) })) + }) +}) diff --git a/forge-web/lib/storage/store.ts b/forge-web/lib/storage/store.ts new file mode 100644 index 000000000..bdce94349 --- /dev/null +++ b/forge-web/lib/storage/store.ts @@ -0,0 +1,155 @@ +/** + * The browser's storage configuration: named profiles, the default browser-push policy, and + * per-repo overrides. All of it lives in ONE blob sealed in the vault (`lib/auth/vault.ts` + * `writeStorageBlob`): the profiles' secrets must never touch localStorage or plaintext + * IndexedDB, and keeping the non-secret parts next to them means there is exactly one place + * to read, one to forget, and nothing to reconcile. Reading needs an unlocked session, which + * every push needs anyway. + * + * What is stored is parsed with zod on the way out: a tampered or foreign blob fails to + * decrypt (AES-GCM with the network and identity as additional data) and, if it did not, + * would still not reach the uploader unvalidated. + */ + +import { z } from 'zod' + +import type { Network } from '../constants' +import { discardStorageBlob, readStorageBlob, writeStorageBlob } from '../auth/vault' +import { + policySchema, + renameInPolicy, + profilePublicSchema, + profileSecretsSchema, + type StoragePolicy, + type StorageProfile, +} from './profiles' + +/** Everything this browser knows about the user's storage. */ +export interface StorageConfig { + readonly version: 1 + readonly profiles: readonly StorageProfile[] + /** The policy browser pushes use when a repo has no override. */ + readonly defaultPolicy: StoragePolicy | null + /** repoId → the policy for browser pushes to that repo. */ + readonly repoPolicies: Readonly> + /** Profile name → the last browser test: when, and whether every row passed. */ + readonly lastTests: Readonly> +} + +export const EMPTY_STORAGE_CONFIG: StorageConfig = { + version: 1, + profiles: [], + defaultPolicy: null, + repoPolicies: {}, + lastTests: {}, +} + +const configSchema = z.object({ + version: z.literal(1), + profiles: z + .array(z.object({ name: z.string().max(64), settings: profilePublicSchema, secrets: profileSecretsSchema })) + .max(32), + defaultPolicy: policySchema.nullable(), + repoPolicies: z.record(z.string().max(64), policySchema), + lastTests: z.record(z.string().max(64), z.object({ at: z.number(), ok: z.boolean() })), +}) + +/** The stored configuration (empty when none is stored). Needs the vault unlocked. */ +export async function loadStorageConfig(network: Network, identityId: string): Promise { + const raw = await readStorageBlob(network, identityId) + if (raw === null) return EMPTY_STORAGE_CONFIG + const parsed = configSchema.safeParse(raw) + if (!parsed.success) throw new Error('the stored storage settings are not in a shape this app understands') + return parsed.data as StorageConfig +} + +/** Seal `config` in the vault. Needs the vault unlocked. */ +export async function saveStorageConfig(network: Network, identityId: string, config: StorageConfig): Promise { + const parsed = configSchema.parse(config) + await writeStorageBlob(network, identityId, parsed) +} + +/** + * Delete stored storage settings this key cannot open (sealed under an earlier key, or in a + * shape this app does not understand). The profiles in them are lost: add them again. + */ +export function discardStorageConfig(network: Network, identityId: string): Promise { + return discardStorageBlob(network, identityId) +} + +/** + * `config` with the profile `from` replaced by `profile`: every policy that named it follows + * a rename, as does its last test result. + */ +export function withRenamedProfile(config: StorageConfig, from: string, profile: StorageProfile): StorageConfig { + const to = profile.name + const repoPolicies: Record = {} + for (const [repo, p] of Object.entries(config.repoPolicies)) repoPolicies[repo] = renameInPolicy(p, from, to) + const lastTests = { ...config.lastTests } + delete lastTests[from] + return withProfile( + { + ...config, + profiles: config.profiles.filter((p) => p.name !== from), + defaultPolicy: config.defaultPolicy ? renameInPolicy(config.defaultPolicy, from, to) : null, + repoPolicies, + lastTests, + }, + profile, + ) +} + +/** `config` with `profile` added or replaced (by name). */ +export function withProfile(config: StorageConfig, profile: StorageProfile): StorageConfig { + const profiles = [...config.profiles.filter((p) => p.name !== profile.name), profile].sort((a, b) => a.name.localeCompare(b.name)) + return { ...config, profiles } +} + +/** + * `config` without the profile `name`: it also leaves every policy (a policy naming a profile + * that no longer exists would fail every push). A policy left with no targets is removed. + */ +export function withoutProfile(config: StorageConfig, name: string): StorageConfig { + const prune = (p: StoragePolicy | null): StoragePolicy | null => { + if (p === null) return null + const targets = p.targets.filter((t) => t !== name) + if (targets.length === 0) return null + return { ...p, targets, replicas: Math.min(p.replicas, targets.length) } + } + const repoPolicies: Record = {} + for (const [repo, p] of Object.entries(config.repoPolicies)) { + const next = prune(p) + if (next) repoPolicies[repo] = next + } + const lastTests = { ...config.lastTests } + delete lastTests[name] + return { + ...config, + profiles: config.profiles.filter((p) => p.name !== name), + defaultPolicy: prune(config.defaultPolicy), + repoPolicies, + lastTests, + } +} + +/** The policy browser pushes to `repoId` use: its override, else the default, else none. */ +export function policyForRepo(config: StorageConfig, repoId: string): StoragePolicy | null { + return config.repoPolicies[repoId] ?? config.defaultPolicy +} + +/** `config` with `repoId`'s override set (or cleared with null). */ +export function withRepoPolicy(config: StorageConfig, repoId: string, policy: StoragePolicy | null): StorageConfig { + const repoPolicies = { ...config.repoPolicies } + if (policy === null) delete repoPolicies[repoId] + else repoPolicies[repoId] = policy + return { ...config, repoPolicies } +} + +/** Why a policy cannot be used with `config`'s profiles, or null. */ +export function policyProblem(config: StorageConfig, policy: StoragePolicy): string | null { + if (policy.targets.length === 0) return 'choose at least one place to store packs' + const missing = policy.targets.filter((t) => !config.profiles.some((p) => p.name === t)) + if (missing.length > 0) return `no storage profile named ${missing.join(', ')}` + if (policy.replicas > policy.targets.length) return `${policy.replicas} copies are required but only ${policy.targets.length} places are chosen` + return null +} diff --git a/forge-web/lib/storage/upload.live.test.ts b/forge-web/lib/storage/upload.live.test.ts new file mode 100644 index 000000000..8025128e2 --- /dev/null +++ b/forge-web/lib/storage/upload.live.test.ts @@ -0,0 +1,106 @@ +/** + * Live browser-upload path — SKIPPED by default (devnet writes + a local MinIO). + * + * docker compose -f infra/docker-compose.yml up -d minio minio-init + * FORGE_LIVE=1 FORGE_LIVE_PUBLIC_URL=https://…/forge-byo NEXT_PUBLIC_NETWORK=devnet NEXT_PUBLIC_DEVNET_NAME=moutai \ + * pnpm exec vitest run lib/storage/upload.live.test.ts + * + * As the moutai MAINTAINER (a maintainer of its own `forge-v2-empty`, the fixture no push has + * touched): store a small artifact in the `forge-byo` bucket with SigV4 (the same code the + * browser runs, on Node's WebCrypto + fetch), verified by re-read; write its `packManifest` + * (kind 2, a flatIndex slot nothing reads for browsing); then read the manifest back from + * Platform and fetch + hash the bytes through the web reader (`loadArtifactBytesProgress`), + * as any browser would. A rerun is idempotent (content-addressed key, the signer's own + * manifest slot). Spend: one manifest document. + */ + +import { readFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' + +import { describe, expect, it } from 'vitest' + +import { DEFAULT_NETWORK, NETWORKS } from '../constants' +import { isPublicHttpsUrl } from '../net' +import { evoSdkService, type WriteAuth } from '../sdk' +import { parseIdentityFileText } from '../auth' +import { readV2PackCopies } from '../repo' +import { loadArtifactBytesProgress } from '../view/browse-source' +import { retryWhileMissing } from '../view/retry' +import type { V2RepoRef } from '../repo/contract' +import { storeAndRecordPack, policyFor, type StorageProfile } from './index' +import { sha256Hex } from './sigv4' + +/** + * The bucket's PUBLIC https address: a manifest records only addresses anyone can read, so a + * bare local MinIO does not qualify. Expose it for the run, e.g. + * `cloudflared tunnel --url http://127.0.0.1:9000` → FORGE_LIVE_PUBLIC_URL=https://.trycloudflare.com/forge-byo + */ +const PUBLIC_URL = (process.env['FORGE_LIVE_PUBLIC_URL'] ?? '').replace(/\/+$/, '') +const LIVE = process.env['FORGE_LIVE'] === '1' && DEFAULT_NETWORK === 'devnet' && isPublicHttpsUrl(PUBLIC_URL) +const MAINTAINER_FILE = join(homedir(), '.config/dash-forge/test-identities/devnet-moutai/MAINTAINER.identity.json') +const EMPTY_REPO = 'F9puk5NBQyySFbgj9yUfubdXAVv1YV7Zk2KDvfyHzxLU' + +const MINIO: StorageProfile = { + name: 'minio', + settings: { + kind: 's3', + provider: 'minio', + endpoint: 'http://127.0.0.1:9000', + region: 'us-east-1', + bucket: 'forge-byo', + pathStyle: true, + publicUrl: PUBLIC_URL, + prefix: 'web-live', + }, + secrets: { accessKeyId: 'minioadmin', secretAccessKey: 'minioadmin' }, +} + +describe.skipIf(!LIVE)('live browser upload to MinIO + manifest on moutai', () => { + it( + 'stores, verifies, records, and reads back through the web reader', + async () => { + const parsed = parseIdentityFileText(readFileSync(MAINTAINER_FILE, 'utf8')) + const auth: WriteAuth = { identityId: parsed.identityId, network: 'devnet', getSigningKeyWif: () => parsed.signingKeyWif } + const v2 = NETWORKS.devnet.v2 + if (!v2) throw new Error('no forge-v2 deployment on this devnet') + await evoSdkService.initialize({ network: 'devnet', contractIds: [v2.core, v2.collab], timeoutMs: 20000 }) + const sdk = evoSdkService.getSdk() + const repo: V2RepoRef = { kind: 'v2', forge: v2, repoId: EMPTY_REPO, ownerId: parsed.identityId, name: 'forge-v2-empty', visibility: 'public' } + + // One artifact per public URL: the signer's manifest slot for a pack is permanent, so a + // rerun with the same bytes and URL finds its own manifest and writes nothing. + const bytes = new TextEncoder().encode(`forge-web live upload artifact ${EMPTY_REPO} ${PUBLIC_URL}\n`) + const hash = await sha256Hex(bytes) + const events: string[] = [] + const { stored, manifest } = await storeAndRecordPack( + sdk, + auth, + repo, + bytes, + { kind: 2, objectCount: 0 }, + { + policy: policyFor(['minio'], 'one'), + profiles: [MINIO], + confirmPlatform: async () => false, + onStep: (e) => events.push(`${e.target}:${e.phase}`), + intent: `live-upload:${hash}`, + }, + ) + // eslint-disable-next-line no-console + console.log('stored', { uris: stored.uris, manifest: manifest.documentId, cost: manifest.cost.dash, events }) + expect(stored.storage).toBe(1) + expect(stored.uris[0]).toBe(`${PUBLIC_URL}/web-live/packs/${hash}.pack`) + expect(stored.uris[1]).toBe(`s3://forge-byo/web-live/packs/${hash}.pack`) + expect(events).toEqual(['minio:start', 'minio:done']) + + // The node answering the read may be a block behind the one that confirmed the write. + const copy = await retryWhileMissing(() => readV2PackCopies(sdk, repo, hash, 2), 8) + expect(copy, 'the manifest should read back from Platform').not.toBeNull() + expect(copy?.uris).toEqual(stored.uris) + const got = await loadArtifactBytesProgress(sdk, repo, copy!) + expect(await sha256Hex(got)).toBe(hash) + }, + 180_000, + ) +}) diff --git a/forge-web/lib/storage/upload.ts b/forge-web/lib/storage/upload.ts new file mode 100644 index 000000000..e35b13591 --- /dev/null +++ b/forge-web/lib/storage/upload.ts @@ -0,0 +1,336 @@ +/** + * Browser uploads of pack artifacts under the user's storage policy: the web counterpart of + * forge-core `storage/targets.rs` (`replicate`, `ExternalTarget`) and `StoredArtifact`. + * + * Order of events, the same guarantee the CLI gives: the bytes are stored AND verified on at + * least `replicas` targets before this resolves, and only then may the caller write the + * `packManifest` (and after it, refs). Nothing is written to Platform without the user first + * agreeing to its price ({@link StoreOptions.confirmPlatform}) — asked once, before the first + * chunk, whether Platform is a target of the policy, the fallback, or the only place. + * + * A copy counts as confirmed only when the URLs it records are readable by anyone, as in the + * CLI (`ExternalTarget::store` re-reads through the first recorded URI): + * - S3: the signed API re-read (whole object up to 16 MiB re-hashed; larger: size plus + * byte-exact head and tail windows), then an anonymous ranged read of the head and tail + * through the public URL — the one readers use; + * - IPFS: kubo's CID must equal the local derivation and be pinned, then a re-read through the + * node's gateway when one is configured, then an anonymous read through the public gateway + * when one is recorded. + * Every URL recorded must be a public https URL (`lib/net.ts`); a profile whose public address + * is not is refused before any upload. + */ + +import type { EvoSDK } from '@dashevo/evo-sdk' + +import type { WriteAuth } from '../sdk' +import { estimateChunkCredits } from '../sdk/cost' +import { isRecordableUri } from '../net' +import type { V2RepoRef } from '../repo/contract' +import { manifestUrisProblem, putPlatformChunks } from '../repo/push' +import { addVerified, gatewayUrl, remotePin } from './ipfs' +import { artifactKey, profileProblem, publishProblem, type StoragePolicy, type StorageProfile } from './profiles' +import { getObject, getPublic, headObject, putObject, publicObjectUrl, s3Uri, type S3Settings } from './s3' +import { sha256Hex } from './sigv4' +import { bytesEqual, errText, timedFetch } from './util' + +/** Artifacts up to this size are verified by a full re-download and SHA-256. */ +export const FULL_VERIFY_MAX = 16 * 1024 * 1024 +const EDGE_WINDOW = 64 * 1024 + +/** A target that did not confirm. */ +export interface TargetFailure { + readonly target: string + readonly reason: string +} + +/** Progress of one target. */ +export type UploadEvent = + | { readonly target: string; readonly phase: 'start' } + | { readonly target: string; readonly phase: 'progress'; readonly done: number; readonly total: number } + | { readonly target: string; readonly phase: 'done'; readonly uris: readonly string[] } + | { readonly target: string; readonly phase: 'failed'; readonly reason: string } + +/** Where an artifact now lives: what its `packManifest` records. */ +export interface StoredArtifact { + readonly packHash: string + readonly sizeBytes: number + /** 0 when a Platform copy exists (chunks), else 1. */ + readonly storage: 0 | 1 + readonly chunkCount: number + /** `platform://` first, then public https URLs, then the rest (`ipfs://`, `s3://`). */ + readonly uris: readonly string[] + /** Targets that confirmed, in policy order. */ + readonly confirmed: readonly string[] + readonly failures: readonly TargetFailure[] +} + +/** The price of putting `bytes` on Platform, asked before paying it. */ +export interface PlatformQuestion { + readonly bytes: number + readonly estimateCredits: number + /** Why Platform is being used: a target of the policy, the fallback, or nothing configured. */ + readonly reason: string +} + +/** Fewer than the required targets confirmed. */ +export class ReplicationError extends Error { + constructor( + readonly required: number, + readonly confirmed: readonly string[], + readonly failures: readonly TargetFailure[], + /** Whether any Platform chunks were written before the policy failed (they stay: chunks are permanent). */ + readonly platformWritten: boolean, + ) { + super( + `storage policy not met: ${confirmed.length} of ${required} required target(s) confirmed` + + failures.map((f) => `; ${f.target}: ${f.reason}`).join('') + + (platformWritten + ? '. Some pack chunks were written to Platform before the push stopped; they are kept, and a retry does not pay for them again. No manifest or ref was written.' + : '. Nothing was written to Platform.'), + ) + this.name = 'ReplicationError' + } +} + +/** The user declined to store on Platform. */ +export class PlatformDeclinedError extends Error { + constructor() { + super('Nothing was stored: storing on Platform was declined. Configure storage in Settings → Storage and try again.') + this.name = 'PlatformDeclinedError' + } +} + +/** The byte windows compared at both ends of an artifact (the whole of a small one). */ +function edgeWindows(size: number): (readonly [number, number])[] { + if (size <= 2 * EDGE_WINDOW) return [[0, size]] + return [ + [0, EDGE_WINDOW], + [size - EDGE_WINDOW, size], + ] +} + +/** Re-read an S3 copy through the signed API and check it is exactly `bytes`. */ +async function verifyS3(s: S3Settings, p: StorageProfile, key: string, bytes: Uint8Array, hashHex: string): Promise { + if (bytes.length <= FULL_VERIFY_MAX) { + const got = await getObject(s, p.secrets, key) + if ((await sha256Hex(got)) !== hashHex) throw new Error(`re-read of ${key} returned ${got.length} bytes that do not hash to the artifact`) + return + } + const size = await headObject(s, p.secrets, key) + if (size !== bytes.length) throw new Error(`re-read of ${key}: object missing or wrong size (${size} ≠ ${bytes.length})`) + for (const [start, end] of edgeWindows(bytes.length)) { + const got = await getObject(s, p.secrets, key, `bytes=${start}-${end - 1}`) + if (!bytesEqual(got, bytes.subarray(start, end))) throw new Error(`re-read of ${key} bytes ${start}..${end} differ from the artifact`) + } +} + +/** Read the head and tail of a copy anonymously, the way every reader will. */ +async function verifyPublic(label: string, read: (range: string) => Promise<{ status: number; bytes: Uint8Array }>, bytes: Uint8Array): Promise { + for (const [start, end] of edgeWindows(bytes.length)) { + let got: { status: number; bytes: Uint8Array } + try { + got = await read(`bytes=${start}-${end - 1}`) + } catch (e) { + throw new Error(`${label} is not readable by others from a browser (${errText(e)}): check public access and CORS`) + } + if (got.status !== 206 && got.status !== 200) throw new Error(`${label} answered HTTP ${got.status} to an anonymous read: the copy is not public`) + const slice = got.status === 200 ? got.bytes.subarray(start, end) : got.bytes + if (!bytesEqual(slice, bytes.subarray(start, end))) throw new Error(`${label} served bytes ${start}..${end} that differ from the artifact`) + } +} + +/** Store and verify on one external profile; the URIs its manifest entry records. */ +async function storeExternal(p: StorageProfile, bytes: Uint8Array, hashHex: string): Promise { + const problem = profileProblem(p) ?? publishProblem(p) + if (problem) throw new Error(`the profile is not usable: ${problem}`) + const s = p.settings + if (s.kind === 's3') { + const key = artifactKey(s, hashHex) + const existing = await headObject(s, p.secrets, key).catch(() => null) + if (existing !== bytes.length) await putObject(s, p.secrets, key, bytes) + try { + await verifyS3(s, p, key, bytes, hashHex) + } catch (first) { + // A same-size but corrupt object at a content-addressed key would fail every push: + // re-upload once, unconditionally, and check again. + await putObject(s, p.secrets, key, bytes) + try { + await verifyS3(s, p, key, bytes, hashHex) + } catch (second) { + throw new Error(`${errText(second)} (also after a re-upload; first attempt: ${errText(first)})`) + } + } + await verifyPublic(`the public URL (${new URL(s.publicUrl).host})`, (range) => getPublic(s, key, range), bytes) + return [publicObjectUrl(s, key), s3Uri(s, key)] + } + if (s.kind === 'ipfs-kubo' || s.kind === 'ipfs-pinning-service') { + const cid = await addVerified(s, p.secrets, bytes) + if (s.kind === 'ipfs-pinning-service') await remotePin(s, p.secrets, cid, `dash-forge ${hashHex.slice(0, 16)}`) + if (s.gateway !== '') { + const r = await timedFetch(gatewayUrl(s.gateway, cid), { credentials: 'omit', cache: 'no-store' }) + const got = await r.bytes() + if (!r.resp.ok || (await sha256Hex(got)) !== hashHex) throw new Error(`re-read of ipfs://${cid} through the gateway did not return the artifact`) + } + const uris = [`ipfs://${cid}`] + if (s.publicGateway !== '') { + const url = gatewayUrl(s.publicGateway, cid) + await verifyPublic(`the public gateway (${new URL(url).host})`, async (range) => { + const r = await timedFetch(url, { headers: { Range: range }, credentials: 'omit', cache: 'no-store' }) + return { status: r.resp.status, bytes: await r.bytes() } + }, bytes) + uris.unshift(url) + } + return uris + } + throw new Error('not an external profile') +} + +/** + * Every recorded URI, de-duplicated: the `platform://` locator first (released helpers read + * `uris[0]` of a storage-0 manifest as the chunk locator), then public https URLs, then the + * rest — each group in policy order (parity with forge-core `Replication::uris`). + */ +export function orderUris(groups: readonly (readonly string[])[]): string[] { + const buckets: string[][] = [[], [], []] + for (const uris of groups) { + for (const u of uris) { + const g = u.startsWith('platform://') ? 0 : /^https?:\/\//.test(u) ? 1 : 2 + const bucket = buckets[g] as string[] + if (!bucket.includes(u)) bucket.push(u) + } + } + return buckets.flat() +} + +/** + * Trim to the manifest's `uris` budget (8 × 300 bytes): private `s3://` locators go first + * (readers without that profile cannot use them); then an error, never a silent truncation. + * A URI no reader may fetch (plain http, this machine, a private network) is refused outright: + * it would be published on chain for every reader's browser to request. + */ +export function fitManifestUris(uris: readonly string[]): string[] { + const bad = uris.find((u) => !isRecordableUri(u)) + if (bad !== undefined) throw new Error(`refusing to record ${bad} on chain: only public https addresses can be read by others`) + const fits = (list: readonly string[]): boolean => manifestUrisProblem(list) === null + if (fits(uris)) return [...uris] + const noS3 = uris.filter((u) => !u.startsWith('s3://')) + if (fits(noS3)) return noS3 + throw new Error("the confirmed copies' URIs do not fit a manifest (at most 8, each up to 300 bytes); use shorter public URLs or fewer targets") +} + +/** What {@link storeArtifact} needs besides the bytes. */ +export interface StoreOptions { + readonly policy: StoragePolicy | null + readonly profiles: readonly StorageProfile[] + /** Asked once, with the price, before anything is written to Platform. False stops it. */ + readonly confirmPlatform: (q: PlatformQuestion) => Promise + readonly onStep?: (e: UploadEvent) => void +} + +/** + * Store `bytes` under `policy`, verifying every copy, and return what the manifest records. + * Before the first Platform chunk the user is asked, with the price; declining throws + * {@link PlatformDeclinedError} (nothing configured) or {@link ReplicationError} (the policy + * needed the Platform copy). Fewer than `replicas` confirmations throws {@link ReplicationError}. + */ +export async function storeArtifact( + sdk: EvoSDK, + auth: WriteAuth, + repo: V2RepoRef, + bytes: Uint8Array, + opts: StoreOptions, +): Promise { + const hashHex = await sha256Hex(bytes) + const step = opts.onStep ?? (() => undefined) + const confirmed: { target: string; uris: string[]; platform: boolean }[] = [] + const failures: TargetFailure[] = [] + let chunkCount = 0 + let platformWritten = false + let platformAgreed: boolean | null = null + + const fail = (target: string, e: unknown): void => { + failures.push({ target, reason: errText(e) }) + step({ target, phase: 'failed', reason: errText(e) }) + } + /** Ask once per push; every Platform write goes through this first. */ + const agreePlatform = async (reason: string): Promise => { + if (platformAgreed === null) { + platformAgreed = await opts.confirmPlatform({ bytes: bytes.length, estimateCredits: estimateChunkCredits(bytes.length), reason }) + } + return platformAgreed + } + + const platformCopy = async (target: string): Promise => { + step({ target, phase: 'start' }) + try { + const r = await putPlatformChunks(sdk, auth, repo, bytes, hashHex, (done, total, written) => { + if (written > 0) platformWritten = true + step({ target, phase: 'progress', done, total }) + }) + chunkCount = r.chunkCount + confirmed.push({ target, uris: [r.locator], platform: true }) + step({ target, phase: 'done', uris: [r.locator] }) + } catch (e) { + fail(target, e) + throw e + } + } + + const policy = opts.policy + if (policy === null || policy.targets.length === 0) { + if (!(await agreePlatform('No storage is configured for browser pushes to this repo.'))) throw new PlatformDeclinedError() + await platformCopy('platform') + } else { + const byName = new Map(opts.profiles.map((p) => [p.name, p])) + const targets = policy.targets.map((name) => ({ name, profile: byName.get(name) })) + const external = targets.filter((t) => t.profile?.settings.kind !== 'platform') + const onChain = targets.filter((t) => t.profile?.settings.kind === 'platform') + await Promise.all( + external.map(async ({ name, profile }) => { + step({ target: name, phase: 'start' }) + try { + if (!profile) throw new Error('no such storage profile in this browser') + const uris = await storeExternal(profile, bytes, hashHex) + confirmed.push({ target: name, uris, platform: false }) + step({ target: name, phase: 'done', uris }) + } catch (e) { + fail(name, e) + } + }), + ) + const notMet = (): ReplicationError => new ReplicationError(policy.replicas, confirmed.map((c) => c.target), failures, platformWritten) + // Platform targets only if they can still make up the policy: never pay for chunks whose + // push is about to fail anyway — and never without the user's go-ahead on the price. + for (const [i, { name }] of onChain.entries()) { + if (confirmed.length + (onChain.length - i) < policy.replicas) break + if (!(await agreePlatform(`Your storage policy for this repo includes Dash Platform (${name}).`))) { + fail(name, new Error('storing on Platform was declined')) + throw notMet() + } + await platformCopy(name).catch(() => undefined) + } + if (confirmed.length < policy.replicas) { + if (!policy.platformFallback || confirmed.some((c) => c.platform)) throw notMet() + if (!(await agreePlatform(`Your storage did not confirm (${failures.map((f) => f.target).join(', ')}).`))) throw notMet() + try { + await platformCopy('platform (fallback)') + } catch { + throw notMet() + } + } + } + + // Keep policy order within each URI group. + const order = policy?.targets ?? [] + confirmed.sort((a, b) => order.indexOf(a.target) - order.indexOf(b.target)) + const hasPlatform = confirmed.some((c) => c.platform) + return { + packHash: hashHex, + sizeBytes: bytes.length, + storage: hasPlatform ? 0 : 1, + chunkCount: hasPlatform ? chunkCount : 0, + uris: fitManifestUris(orderUris(confirmed.map((c) => c.uris))), + confirmed: confirmed.map((c) => c.target), + failures, + } +} diff --git a/forge-web/lib/storage/util.ts b/forge-web/lib/storage/util.ts new file mode 100644 index 000000000..b7fe4c01c --- /dev/null +++ b/forge-web/lib/storage/util.ts @@ -0,0 +1,117 @@ +/** Small helpers shared by the live test (`./probe`) and the upload path (`./upload`). */ + +/** Whether `a` and `b` hold the same bytes. */ +export function bytesEqual(a: Uint8Array, b: Uint8Array): boolean { + return a.length === b.length && a.every((x, i) => x === b[i]) +} + +/** The message of a thrown value, as a row detail or a failure reason. */ +export function errText(e: unknown): string { + return e instanceof Error ? e.message : String(e) +} + +/** A response is expected to begin within this long (a dead or silent host). */ +export const CONNECT_MS = 15_000 +/** A response body may pause this long between chunks before it is abandoned. */ +export const IDLE_MS = 120_000 +/** + * The slowest upload rate assumed when a request carries a body: `fetch` reports no upload + * progress, so the wait for the response grows with the body instead of being one fixed cap. + * 16 KiB/s is well below any usable uplink; a stall still ends within `CONNECT_MS + IDLE_MS` + * plus the body at that rate. + */ +const MIN_UPLOAD_BYTES_PER_S = 16 * 1024 + +/** The request timed out (no answer, or the body stalled). */ +export class TimeoutError extends Error { + constructor(message: string) { + super(message) + this.name = 'TimeoutError' + } +} + +/** A response whose body is read with an idle deadline re-armed on every chunk. */ +export interface TimedResponse { + readonly resp: Response + bytes(): Promise + text(): Promise + /** Done without the body (a HEAD, a status-only answer): clear the deadline, drop the body. */ + discard(): void +} + +/** + * `fetch` with progress deadlines rather than one total cap (forge-core's clients use a 15 s + * connect and a 120 s read timeout): the response must begin within {@link CONNECT_MS} (plus + * the upload time of `uploadBytes` at a conservative rate), then the body may stream for as + * long as it keeps moving, pausing at most {@link IDLE_MS}. + */ +export async function timedFetch( + url: string | URL, + init: RequestInit, + opts: { readonly uploadBytes?: number; readonly signal?: AbortSignal; readonly idleMs?: number } = {}, +): Promise { + const controller = new AbortController() + const idleMs = opts.idleMs ?? IDLE_MS + let timedOut: string | null = null + let timer: ReturnType | undefined + const arm = (ms: number, why: string): void => { + clearTimeout(timer) + timer = setTimeout(() => { + timedOut = why + controller.abort() + }, ms) + } + const onCancel = (): void => controller.abort() + opts.signal?.addEventListener('abort', onCancel) + const uploadMs = Math.ceil(((opts.uploadBytes ?? 0) / MIN_UPLOAD_BYTES_PER_S) * 1000) + arm(CONNECT_MS + uploadMs, `no answer within ${Math.round((CONNECT_MS + uploadMs) / 1000)}s`) + const done = (): void => { + clearTimeout(timer) + opts.signal?.removeEventListener('abort', onCancel) + } + let resp: Response + try { + resp = await fetch(url, { ...init, signal: controller.signal }) + } catch (e) { + done() + if (timedOut !== null) throw new TimeoutError(timedOut) + throw e + } + const bytes = async (): Promise => { + try { + if (resp.body === null) return new Uint8Array(await resp.arrayBuffer()) + const reader = resp.body.getReader() + const parts: Uint8Array[] = [] + let total = 0 + for (;;) { + arm(idleMs, `the answer stalled for ${idleMs / 1000}s`) + const { done: end, value } = await reader.read() + if (end) break + parts.push(value) + total += value.length + } + const out = new Uint8Array(total) + let at = 0 + for (const p of parts) { + out.set(p, at) + at += p.length + } + return out + } catch (e) { + if (timedOut !== null) throw new TimeoutError(timedOut) + throw e + } finally { + done() + } + } + const discard = (): void => { + done() + resp.body?.cancel().catch(() => undefined) + } + return { resp, bytes, text: async () => new TextDecoder().decode(await bytes()), discard } +} + +/** Printable ASCII only: header values (`fetch` throws on anything else, quoting the value). */ +export function isHeaderSafe(value: string): boolean { + return /^[\x20-\x7e]*$/.test(value) +} diff --git a/forge-web/lib/view/browse-fallback.test.ts b/forge-web/lib/view/browse-fallback.test.ts index a59f36fd7..3ec6cee95 100644 --- a/forge-web/lib/view/browse-fallback.test.ts +++ b/forge-web/lib/view/browse-fallback.test.ts @@ -163,7 +163,7 @@ describe('startFallback with external-storage packs', () => { const external = manifestFor(ext.pack, 1, { storage: 1, chunkCount: 0, - uris: ['http://127.0.0.1:9000/forge-byo/pack', 's3://forge-byo/pack'], + uris: ['https://mirror.example/forge-byo/pack', 's3://forge-byo/pack'], createdAt: 2, documentId: 'b', uploader: 'owner', @@ -172,16 +172,16 @@ describe('startFallback with external-storage packs', () => { const calls = stubFetch({}) const ctx = await startFallback(mockSdk(new Map([[platform.packHash, plat.pack]])), repo, [platform, external]) - expect(calls).toEqual(['http://127.0.0.1:9000/forge-byo/pack']) // s3:// is not browser-fetchable + expect(calls).toEqual(['https://mirror.example/forge-byo/pack']) // s3:// is not browser-fetchable expect(ctx.unavailable).toHaveLength(1) expect(ctx.unavailable?.[0]?.packHash).toBe(external.packHash) - expect(ctx.unavailable?.[0]?.hosts).toEqual(['127.0.0.1:9000']) + expect(ctx.unavailable?.[0]?.hosts).toEqual(['mirror.example']) expect(Array.from((await ctx.reader.readObject(plat.oid)).bytes)).toEqual( Array.from(new TextEncoder().encode('on-chain content\n')), ) // A needed object from the skipped pack gets a per-view error naming pack and storage. await expect(ctx.reader.readObject(ext.oid)).rejects.toThrow( - new RegExp(`${external.packHash.slice(0, 12)}.*127\\.0\\.0\\.1:9000`), + new RegExp(`${external.packHash.slice(0, 12)}.*mirror\\.example`), ) // The trust ledger reports the gap: content is partial, never verified. const checks = contentChecks(repo.contractId) @@ -234,7 +234,7 @@ describe('startFallback with external-storage packs', () => { const ext = blobPack('gone\n') const platform = manifestFor(plat.pack, 1, { createdAt: 1, documentId: 'a' }) const external = (documentId: string): PackManifest => - manifestFor(ext.pack, 1, { storage: 1, uris: ['http://127.0.0.1:9000/p'], createdAt: 2, documentId }) + manifestFor(ext.pack, 1, { storage: 1, uris: ['https://mirror.example/p'], createdAt: 2, documentId }) stubFetch({}) const ctx = await startFallback( mockSdk(new Map([[platform.packHash, plat.pack]])), @@ -283,7 +283,7 @@ describe('startFallback with external-storage packs', () => { it('fails with the reasons when no live pack at all could be fetched', async () => { const ext = blobPack('x\n') - const external = manifestFor(ext.pack, 1, { storage: 1, uris: ['http://127.0.0.1:9000/p'] }) + const external = manifestFor(ext.pack, 1, { storage: 1, uris: ['https://mirror.example/p'] }) stubFetch({}) await expect( startFallback(mockSdk(new Map()), { kind: 'v1', contractId: 'fallback-none', ownerId: 'owner', name: '' }, [external]), @@ -338,13 +338,13 @@ describe('startFallback with external-storage packs', () => { const platform = manifestFor(plat.pack, 1, { createdAt: 0, documentId: 'a' }) const dead = (i: number): PackManifest => { const p = blobPack(`gone ${i}\n`) - return manifestFor(p.pack, 1, { storage: 1, uris: ['http://127.0.0.1:9000/p'], createdAt: i + 1, documentId: `x${i}` }) + return manifestFor(p.pack, 1, { storage: 1, uris: ['https://mirror.example/p'], createdAt: i + 1, documentId: `x${i}` }) } const calls = stubFetch({}) const sdk = mockSdk(new Map([[platform.packHash, plat.pack]])) await startFallback(sdk, { kind: 'v1', contractId: 'fallback-dead', ownerId: 'o', name: '' }, [platform, dead(1)]) await startFallback(sdk, { kind: 'v1', contractId: 'fallback-dead-2', ownerId: 'o', name: '' }, [platform, dead(2)]) - expect(calls).toEqual(['http://127.0.0.1:9000/p']) + expect(calls).toEqual(['https://mirror.example/p']) }) it('cancels external downloads when an on-chain pack fails', async () => { @@ -378,12 +378,12 @@ describe('startFallback with external-storage packs', () => { const packA = packFrame(concat(objHeader(T_BLOB, base.length), zlibSync(base))) const delta = copyInsertDelta(base.length, target.length, 40, new TextEncoder().encode('cat\n')) const packB = packFrame(concat(objHeader(T_REF_DELTA, delta.length), hexToBytes(baseOid), zlibSync(delta))) - const a = manifestFor(packA, 1, { storage: 1, uris: ['http://127.0.0.1:9000/a'], createdAt: 0, documentId: 'a' }) + const a = manifestFor(packA, 1, { storage: 1, uris: ['https://mirror.example/a'], createdAt: 0, documentId: 'a' }) const b = manifestFor(packB, 1, { createdAt: 1, documentId: 'b' }) stubFetch({}) await expect( startFallback(mockSdk(new Map([[b.packHash, packB]])), { kind: 'v1', contractId: 'fallback-thin', ownerId: 'o', name: '' }, [a, b]), - ).rejects.toThrow(new RegExp(`${a.packHash.slice(0, 12)}.*127\\.0\\.0\\.1:9000`)) + ).rejects.toThrow(new RegExp(`${a.packHash.slice(0, 12)}.*mirror\\.example`)) }) it('expires a partial clone so a later view retries the mirrors', async () => { diff --git a/forge-web/lib/view/browse-source.ts b/forge-web/lib/view/browse-source.ts index 4aee1e3a8..0c877f961 100644 --- a/forge-web/lib/view/browse-source.ts +++ b/forge-web/lib/view/browse-source.ts @@ -42,6 +42,7 @@ import { type V2RepoRef, } from '../repo' import { base64ToBytes, queryDocumentsWithProof } from '../sdk' +import { isPublicHttpsUrl } from '../net' import { externalSourceName, noteContentCheck, objectObserver } from './content-checks' import { describePack, readGateways } from './storage-status' @@ -286,8 +287,10 @@ export function externalFetchUrls( ): string[] { const out: string[] = [] for (const uri of uris) { - if (/^https?:\/\//i.test(uri)) out.push(uri) - const ipfs = /^ipfs:\/\/(.+)$/i.exec(uri) + // A manifest is written by whoever pushed: a URL at this machine, a private network or + // over plain http would have every reader's browser request its own local services. + if (isPublicHttpsUrl(uri)) out.push(uri) + const ipfs = /^ipfs:\/\/([A-Za-z0-9]+)$/.exec(uri) if (ipfs !== null) { for (const gw of gateways) out.push(`${gw.replace(/\/+$/, '')}/ipfs/${ipfs[1]}`) } diff --git a/forge-web/tailwind.config.js b/forge-web/tailwind.config.js index 60b0ebc89..491713f20 100644 --- a/forge-web/tailwind.config.js +++ b/forge-web/tailwind.config.js @@ -51,7 +51,11 @@ module.exports = { DEFAULT: '#d97706', // degraded availability — icons, borders, tints; text on dark 700: '#b45309', // TEXT on light surfaces (5.02:1 on white; the base value is 3.19:1) }, - danger: '#dc2626', // force-push, delete, failed verification + danger: { + DEFAULT: '#dc2626', // force-push, delete, failed verification: icons, borders, tints, text on light + 400: '#f87171', // TEXT on dark surfaces (the base value is 4.01:1 on anvil-950) + 700: '#b91c1c', // TEXT on light surfaces (the base value is 4.4:1 on anvil-100) + }, // Dash brand blue — identity/credits/network UI only. The brand value is for fills, // tints and icons; it is under 4.5:1 as TEXT on every surface in both themes, so // text uses the WCAG AA shades: `text-dash-600 dark:text-dash-400` (the same