From b88293c612407839b2bd3d54a2c59ef6d33e034c Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 26 Sep 2026 15:53:44 -0500 Subject: [PATCH 1/5] feat(web): storage wizard and browser uploads to your own storage Adds /settings/storage (ux-dx-spec 3.1): provider tiles for R2, B2, AWS S3, MinIO, IPFS kubo, an IPFS pinning service and Dash Platform; per-provider forms with where-to-find hints; a live browser test (signed PUT, signed GET, anonymous public GET, CORS for Range reads and for PUT, delete probe) with copy-paste CORS fixes per provider; replication choice; the cost card. Credentials are sealed in the vault next to the limited key (AES-GCM under an HKDF of the vault data key), never in localStorage. The upload path (lib/storage) signs S3 requests with SigV4 on WebCrypto, pinned to the forge-core test vectors, adds to kubo with the CID re-derived locally, pins remotely, verifies every copy by re-read, and only then writes the packManifest (lib/repo/push.ts), matching forge-core field for field. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/app/layout.tsx | 6 +- forge-web/app/settings/page.tsx | 10 + forge-web/app/settings/storage/page.tsx | 46 +++ .../components/repo/settings-content.tsx | 7 +- forge-web/components/storage/copy-block.tsx | 33 ++ forge-web/components/storage/cost-card.tsx | 56 +++ forge-web/components/storage/profile-form.tsx | 186 +++++++++ .../storage/repo-storage-policy.tsx | 123 ++++++ forge-web/components/storage/storage-test.tsx | 104 +++++ .../components/storage/storage-wizard.tsx | 370 ++++++++++++++++++ forge-web/components/ui/button.tsx | 2 +- forge-web/e2e/helpers.ts | 2 +- forge-web/e2e/storage-wizard.spec.ts | 133 +++++++ forge-web/hooks/use-storage-config.ts | 45 +++ forge-web/lib/auth/vault.ts | 108 ++++- forge-web/lib/design/contrast.test.ts | 15 + forge-web/lib/repo/push.ts | 203 ++++++++++ forge-web/lib/sdk/cost.ts | 19 + forge-web/lib/storage/cid.test.ts | 28 ++ forge-web/lib/storage/cid.ts | 121 ++++++ forge-web/lib/storage/cors.ts | 83 ++++ forge-web/lib/storage/index.ts | 76 ++++ forge-web/lib/storage/ipfs.ts | 168 ++++++++ forge-web/lib/storage/probe.ts | 233 +++++++++++ forge-web/lib/storage/profiles.ts | 321 +++++++++++++++ forge-web/lib/storage/s3.ts | 155 ++++++++ forge-web/lib/storage/sigv4.test.ts | 160 ++++++++ forge-web/lib/storage/sigv4.ts | 164 ++++++++ forge-web/lib/storage/storage.test.ts | 289 ++++++++++++++ forge-web/lib/storage/store.ts | 124 ++++++ forge-web/lib/storage/upload.live.test.ts | 97 +++++ forge-web/lib/storage/upload.ts | 276 +++++++++++++ forge-web/tailwind.config.js | 6 +- 33 files changed, 3752 insertions(+), 17 deletions(-) create mode 100644 forge-web/app/settings/storage/page.tsx create mode 100644 forge-web/components/storage/copy-block.tsx create mode 100644 forge-web/components/storage/cost-card.tsx create mode 100644 forge-web/components/storage/profile-form.tsx create mode 100644 forge-web/components/storage/repo-storage-policy.tsx create mode 100644 forge-web/components/storage/storage-test.tsx create mode 100644 forge-web/components/storage/storage-wizard.tsx create mode 100644 forge-web/e2e/storage-wizard.spec.ts create mode 100644 forge-web/hooks/use-storage-config.ts create mode 100644 forge-web/lib/repo/push.ts create mode 100644 forge-web/lib/storage/cid.test.ts create mode 100644 forge-web/lib/storage/cid.ts create mode 100644 forge-web/lib/storage/cors.ts create mode 100644 forge-web/lib/storage/index.ts create mode 100644 forge-web/lib/storage/ipfs.ts create mode 100644 forge-web/lib/storage/probe.ts create mode 100644 forge-web/lib/storage/profiles.ts create mode 100644 forge-web/lib/storage/s3.ts create mode 100644 forge-web/lib/storage/sigv4.test.ts create mode 100644 forge-web/lib/storage/sigv4.ts create mode 100644 forge-web/lib/storage/storage.test.ts create mode 100644 forge-web/lib/storage/store.ts create mode 100644 forge-web/lib/storage/upload.live.test.ts create mode 100644 forge-web/lib/storage/upload.ts diff --git a/forge-web/app/layout.tsx b/forge-web/app/layout.tsx index 70df4d7a8..bc106c2a4 100644 --- a/forge-web/app/layout.tsx +++ b/forge-web/app/layout.tsx @@ -15,7 +15,9 @@ export const metadata: Metadata = { // inline bootstrap scripts. // - frame-ancestors is ignored in a CSP; the host must send it as a header (GitHub // Pages cannot; see docs/guides/identity-and-keys.md). -// - connect-src https:/wss:: DAPI endpoints + IPFS/S3/HTTPS pack backends. +// - connect-src https:/wss:: DAPI endpoints + IPFS/S3/HTTPS pack backends. Plain http only to +// this machine (a local kubo node or MinIO, the storage guide's defaults); storage profiles +// refuse http anywhere else (`lib/storage/profiles.ts`). // - worker-src blob:: materialization / search / pack workers run off-main-thread. const CSP = [ "default-src 'self'", @@ -23,7 +25,7 @@ const CSP = [ "style-src 'self' 'unsafe-inline'", "img-src 'self' data: https: blob:", "font-src 'self'", - "connect-src 'self' https: wss:", + "connect-src 'self' https: wss: http://127.0.0.1:* http://localhost:*", "worker-src 'self' blob:", "child-src 'self' blob:", "frame-ancestors 'none'", diff --git a/forge-web/app/settings/page.tsx b/forge-web/app/settings/page.tsx index 4c9f75dfe..89c56b84a 100644 --- a/forge-web/app/settings/page.tsx +++ b/forge-web/app/settings/page.tsx @@ -83,6 +83,16 @@ export default function SettingsPage(): JSX.Element { +
+

Storage

+

+ Your buckets and IPFS nodes for browser pushes, tested from this page, with their keys encrypted in this browser. +

+ + Storage settings → + +
+

This browser's key

diff --git a/forge-web/app/settings/storage/page.tsx b/forge-web/app/settings/storage/page.tsx new file mode 100644 index 000000000..72799e88b --- /dev/null +++ b/forge-web/app/settings/storage/page.tsx @@ -0,0 +1,46 @@ +'use client' + +/** `/settings/storage` — bring your own storage (`ux-dx-spec.md` §3.1). */ + +import Link from 'next/link' +import { HardDrive, MonitorSmartphone } from 'lucide-react' +import { AppShell } from '@/components/app-shell' +import { Button } from '@/components/ui/button' +import { EmptyState } from '@/components/ui/states' +import { StorageWizard } from '@/components/storage/storage-wizard' +import { useAuth } from '@/contexts/auth-context' +import { useUiStore } from '@/hooks/use-ui-store' + +export default function StorageSettingsPage(): JSX.Element { + const { identity } = useAuth() + const openLogin = useUiStore((s) => s.openLogin) + + return ( + +
+
+

+ Settings / Storage +

+

Storage

+

+ We host nothing. Packs go to a bucket or IPFS node you own; Platform keeps only the signed manifest and refs (about 0.0003 DASH a push). Readers check every byte against its on-chain hash, so your storage provides availability, never trust. +

+
+

+ Use a desktop browser for this step: it needs a few long fields and a copy-paste from your provider’s console. +

+ {identity ? ( + + ) : ( + openLogin()}>Sign in} + /> + )} +
+
+ ) +} diff --git a/forge-web/components/repo/settings-content.tsx b/forge-web/components/repo/settings-content.tsx index e6f776c67..2f7a8e1c8 100644 --- a/forge-web/components/repo/settings-content.tsx +++ b/forge-web/components/repo/settings-content.tsx @@ -13,7 +13,7 @@ */ import { useState } from 'react' -import { Fingerprint, ShieldPlus, Snowflake, UserCog } from 'lucide-react' +import { Fingerprint, HardDrive, ShieldPlus, Snowflake, UserCog } from 'lucide-react' import type { RepoHome } from '@/lib/view' import type { Collaborator, V1RepoRef, V2RepoRef } from '@/lib/repo' import { @@ -41,6 +41,7 @@ import { Button } from '@/components/ui/button' import { Field, Input } from '@/components/ui/input' import { ConfirmDialog } from '@/components/confirm-dialog' import { ErrorState, LoadingBlock } from '@/components/ui/states' +import { RepoStoragePolicy } from '@/components/storage/repo-storage-policy' type Role = 'write' | 'maintain' type Action = { kind: 'grant' | 'suspend' | 'revoke'; member: string; role: Role } @@ -266,6 +267,10 @@ function V2Settings({ home, repo }: { home: RepoHome; repo: V2RepoRef }): JSX.El
+
}> + +
+
}> {members.loading ? ( diff --git a/forge-web/components/storage/copy-block.tsx b/forge-web/components/storage/copy-block.tsx new file mode 100644 index 000000000..158dc2f76 --- /dev/null +++ b/forge-web/components/storage/copy-block.tsx @@ -0,0 +1,33 @@ +'use client' + +import { useState } from 'react' +import { Check, Copy } from 'lucide-react' + +/** A multi-line copyable block (a CORS policy, a few shell lines). */ +export function CopyBlock({ text, label }: { text: string; label: string }): JSX.Element { + const [copied, setCopied] = useState(false) + const copy = async (): Promise => { + try { + await navigator.clipboard.writeText(text) + setCopied(true) + setTimeout(() => setCopied(false), 1200) + } catch { + /* insecure context: the text is still selectable */ + } + } + return ( +
+
+        {text}
+      
+ +
+ ) +} diff --git a/forge-web/components/storage/cost-card.tsx b/forge-web/components/storage/cost-card.tsx new file mode 100644 index 000000000..182109af3 --- /dev/null +++ b/forge-web/components/storage/cost-card.tsx @@ -0,0 +1,56 @@ +/** + * The cost comparison card of the storage wizard (`ux-dx-spec.md` §3.1 step 5), always + * visible: what a repo costs with its packs in your bucket versus on Platform. Platform fees + * are the measured per-push manifest + ref costs; bucket prices are the providers' public + * list prices, rounded, and say so. + */ + +import { CREDITS_PER_DASH, estimateChunkCredits } from '@/lib/sdk/cost' +import { dashToUsd, formatDash } from '@/lib/view/format' + +const MIB = 1024 * 1024 +/** Manifest + ref update per push, measured on moutai (≈ 0.0003 DASH). */ +const PER_PUSH_DASH = 0.0003 + +export function CostCard(): JSX.Element { + const platform50 = estimateChunkCredits(50 * MIB) / CREDITS_PER_DASH + const perMib = estimateChunkCredits(MIB) / CREDITS_PER_DASH + const pushes = PER_PUSH_DASH * 10 + const rows: [string, string, string, string][] = [ + ['Your R2 bucket', `manifest + refs ≈ ${formatDash(PER_PUSH_DASH)} DASH / push`, '≈ $0.001', `≈ ${formatDash(pushes)} DASH ≈ ${dashToUsd(pushes)} + $0.00`], + ['Your B2 / S3 bucket', 'same', '≈ $0.0003 / $0.001', `≈ ${formatDash(pushes)} DASH + < $0.01`], + ['Dash Platform', `${formatDash(perMib)} DASH / MiB, permanent`, '0', `≈ ${formatDash(platform50)} DASH ≈ ${dashToUsd(platform50)} first upload, then ${formatDash(perMib)} DASH per pushed MiB`], + ] + return ( +
+

+ What it costs +

+
+ + + + + + + + + + + {rows.map(([where, once, monthly, example]) => ( + + + + + + + ))} + +
WhereOne-time (Platform fees)Monthly50 MiB repo, 10 pushes/month
{where}{once}{monthly}{example}
+
+

+ Platform figures are estimates from measured fees; bucket prices are the providers’ list prices, rounded. USD is indicative. +

+
+ ) +} diff --git a/forge-web/components/storage/profile-form.tsx b/forge-web/components/storage/profile-form.tsx new file mode 100644 index 000000000..f18844eac --- /dev/null +++ b/forge-web/components/storage/profile-form.tsx @@ -0,0 +1,186 @@ +'use client' + +/** + * The per-provider form of the storage wizard (`ux-dx-spec.md` §3.1 step 2): the fields the + * guide names, each with a "where to find this" hint. Secret fields carry a lock and say where + * they go. Values live in component state only until saved into the vault. + */ + +import { useId, useState } from 'react' +import { ExternalLink, Lock } from 'lucide-react' +import { + profileProblem, + providerPreset, + type ProfilePublic, + type ProfileSecrets, + type ProviderId, + type StorageProfile, +} from '@/lib/storage' +import { Input } from '@/components/ui/input' + +const SECRET_NOTE = 'Stored encrypted in this browser only. Never sent to Forge (there is no Forge server) and never written on-chain.' + +type Values = Record + +function initialValues(provider: ProviderId, existing: StorageProfile | null): Values { + const preset = providerPreset(provider) + if (existing) { + const out: Values = { name: existing.name } + for (const [k, v] of [...Object.entries(existing.settings), ...Object.entries(existing.secrets)]) { + if (typeof v === 'string' || typeof v === 'boolean') out[k] = v + } + return out + } + const base: Values = { name: provider === 'aws' ? 'aws' : provider === 'minio' ? 'minio' : provider === 'kubo' ? 'kubo' : provider === 'pinning' ? 'pinning' : `${provider}-main` } + const empty: Values = + preset.kind === 's3' + ? { endpoint: '', region: 'us-east-1', bucket: '', pathStyle: true, publicUrl: '', prefix: '', accessKeyId: '', secretAccessKey: '', sessionToken: '' } + : { api: '', gateway: '', publicGateway: '', pinningEndpoint: '', apiAuth: '', pinningToken: '' } + const defaults: Values = {} + for (const [k, v] of Object.entries(preset.defaults)) if (v !== undefined) defaults[k] = v + return { ...base, ...empty, ...defaults } +} + +const str = (v: string | boolean | undefined): string => (typeof v === 'string' ? v.trim() : '') + +/** The profile the form's values describe. */ +export function profileFromValues(provider: ProviderId, v: Values): StorageProfile { + const preset = providerPreset(provider) + const secret = (k: keyof ProfileSecrets): Partial => (str(v[k]) ? { [k]: str(v[k]) } : {}) + let settings: ProfilePublic + if (preset.kind === 's3') { + settings = { + kind: 's3', + provider: provider as 'r2' | 'b2' | 'aws' | 'minio', + endpoint: str(v['endpoint']).replace(/\/+$/, ''), + region: str(v['region']) || 'us-east-1', + bucket: str(v['bucket']), + pathStyle: v['pathStyle'] !== false, + publicUrl: str(v['publicUrl']).replace(/\/+$/, ''), + prefix: str(v['prefix']), + } + return { name: str(v['name']), settings, secrets: { ...secret('accessKeyId'), ...secret('secretAccessKey'), ...secret('sessionToken') } } + } + if (preset.kind === 'platform') return { name: str(v['name']) || 'platform', settings: { kind: 'platform', provider: 'platform' }, secrets: {} } + settings = { + kind: preset.kind === 'ipfs-pinning-service' ? 'ipfs-pinning-service' : 'ipfs-kubo', + provider: provider as 'kubo' | 'pinning', + api: str(v['api']).replace(/\/+$/, ''), + gateway: str(v['gateway']).replace(/\/+$/, ''), + publicGateway: str(v['publicGateway']).replace(/\/+$/, ''), + pinningEndpoint: str(v['pinningEndpoint']).replace(/\/+$/, ''), + } + return { name: str(v['name']), settings, secrets: { ...secret('apiAuth'), ...secret('pinningToken') } } +} + +interface FieldSpec { + readonly key: string + readonly label: string + readonly secret?: boolean + readonly optional?: boolean + readonly placeholder?: string +} + +function fieldsFor(provider: ProviderId): FieldSpec[] { + const kind = providerPreset(provider).kind + if (kind === 's3') { + return [ + { key: 'name', label: 'Profile name' }, + { key: 'endpoint', label: 'S3 endpoint' }, + { key: 'region', label: 'Region' }, + { key: 'bucket', label: 'Bucket' }, + { key: 'publicUrl', label: 'Public URL', placeholder: 'https://pub-….r2.dev' }, + { key: 'prefix', label: 'Key prefix', optional: true, placeholder: 'forge/' }, + { key: 'accessKeyId', label: 'Access key id', secret: true }, + { key: 'secretAccessKey', label: 'Secret access key', secret: true }, + { key: 'sessionToken', label: 'Session token (temporary credentials)', secret: true, optional: true }, + ] + } + if (kind === 'platform') return [] + const f: FieldSpec[] = [ + { key: 'name', label: 'Profile name' }, + { key: 'api', label: 'kubo RPC API' }, + { key: 'gateway', label: 'Gateway (verification re-read)', optional: true }, + { key: 'publicGateway', label: 'Public gateway (recorded for browsers)', optional: true }, + { key: 'apiAuth', label: 'API Authorization header', secret: true, optional: true, placeholder: 'Basic …' }, + ] + if (kind === 'ipfs-pinning-service') { + f.push({ key: 'pinningEndpoint', label: 'Pinning Service API URL' }, { key: 'pinningToken', label: 'Pinning access token', secret: true }) + } + return f +} + +export function ProfileForm({ + provider, + existing, + onChange, +}: { + provider: ProviderId + existing: StorageProfile | null + /** The profile the form currently describes, with why it is not usable yet (or null). */ + onChange: (profile: StorageProfile, problem: string | null) => void +}): JSX.Element { + const preset = providerPreset(provider) + const [values, setValues] = useState(() => initialValues(provider, existing)) + const idBase = useId() + const set = (k: string, v: string | boolean): void => { + const next = { ...values, [k]: v } + setValues(next) + const p = profileFromValues(provider, next) + onChange(p, profileProblem(p)) + } + + return ( +
+ {fieldsFor(provider).map((f) => { + const hint = preset.hints[f.key] + const id = `${idBase}-${f.key}` + return ( +
+ + set(f.key, e.target.value)} + aria-describedby={hint || f.secret ? `${id}-hint` : undefined} + /> + {hint || f.secret ? ( +

+ {f.secret ? SECRET_NOTE : null} + {f.secret && hint ? ' ' : null} + {hint ? ( + <> + {hint.text} + {hint.href ? ( + <> + {' '} + + where to find this + + + ) : null} + + ) : null} +

+ ) : null} +
+ ) + })} + {preset.kind === 's3' ? ( + + ) : null} +
+ ) +} diff --git a/forge-web/components/storage/repo-storage-policy.tsx b/forge-web/components/storage/repo-storage-policy.tsx new file mode 100644 index 000000000..72299978f --- /dev/null +++ b/forge-web/components/storage/repo-storage-policy.tsx @@ -0,0 +1,123 @@ +'use client' + +/** + * Repo Settings → Storage for browser pushes: which of the viewer's own storage profiles this + * repo's browser pushes (merges, fork updates, release assets) use, overriding the default + * from `/settings/storage`. Local to this browser: the credentials are, so the choice is too + * (the CLI keeps its equivalent in git config, `dash.storage`). + */ + +import { useState } from 'react' +import Link from 'next/link' +import { HardDrive } from 'lucide-react' +import { choiceOf, policyFor, policyForRepo, policyProblem, withRepoPolicy, type ReplicationChoice } from '@/lib/storage' +import { useStorageConfig } from '@/hooks/use-storage-config' +import { Button } from '@/components/ui/button' + +export function RepoStoragePolicy({ repoId }: { repoId: string }): JSX.Element { + const { config, storable, save, error } = useStorageConfig() + const override = config?.repoPolicies[repoId] ?? null + const effective = config ? policyForRepo(config, repoId) : null + const [targets, setTargets] = useState(null) + const [choice, setChoice] = useState(null) + const [msg, setMsg] = useState(null) + + if (error) return

{error}

+ if (!config) { + return ( +

+ Sign in to choose where your browser pushes to this repo go. Settings stay in this browser. +

+ ) + } + if (config.profiles.length === 0) { + return ( +

+ No storage set up in this browser: browser pushes here would store packs on Platform at ~0.28 DASH/MiB, asking first.{' '} + Set up storage → +

+ ) + } + const shown = targets ?? effective?.targets.slice() ?? [] + const shownChoice = choice ?? (effective ? choiceOf(effective) : 'one') + const policy = shown.length > 0 ? policyFor(shown, shownChoice) : null + const problem = policy ? policyProblem(config, policy) : null + + return ( +
+

+ + + Where your browser pushes to this repo store packs.{' '} + {override ? 'This repo has its own choice.' : effective ? 'Using your default.' : 'No default is set.'} Only you see this: it lives in this browser with your storage keys. + +

+
+ Storage for this repo + {config.profiles.map((p) => ( + + ))} +
+ +
+ + {override ? ( + + ) : null} + {problem && shown.length > 0 ? {problem} : null} + {msg ? {msg} : null} +
+
+ ) +} diff --git a/forge-web/components/storage/storage-test.tsx b/forge-web/components/storage/storage-test.tsx new file mode 100644 index 000000000..435b731e4 --- /dev/null +++ b/forge-web/components/storage/storage-test.tsx @@ -0,0 +1,104 @@ +'use client' + +/** + * The live test rows of the storage wizard (`ux-dx-spec.md` §3.1 step 3). Each row settles to + * ok / FAIL / skipped with a plain reason; a CORS failure expands into the copy-paste fix for + * the provider, prefilled with the bucket and this app's origin, with a Re-test button. + */ + +import { useState } from 'react' +import { AlertTriangle, CheckCircle2, CircleDashed, Loader2, MinusCircle, RotateCw, XCircle } from 'lucide-react' +import { + IPFS_ROWS, + S3_ROWS, + corsFix, + probeProfile, + type ProbeRow, + type RowId, + type RowState, + type StorageProfile, +} from '@/lib/storage' +import { Button } from '@/components/ui/button' +import { CopyBlock } from '@/components/storage/copy-block' +import { cn } from '@/lib/utils' + +const META: Readonly> = { + pending: { word: 'waiting', klass: 'text-anvil-500 dark:text-anvil-400', Icon: CircleDashed }, + running: { word: 'testing', klass: 'text-anvil-500 dark:text-anvil-400', Icon: Loader2 }, + ok: { word: 'ok', klass: 'text-verify', Icon: CheckCircle2 }, + fail: { word: 'FAIL', klass: 'text-danger-700 dark:text-danger-400', Icon: XCircle }, + skipped: { word: 'skipped', klass: 'text-anvil-500 dark:text-anvil-400', Icon: MinusCircle }, +} + +function rowsFor(p: StorageProfile): ProbeRow[] { + const spec = p.settings.kind === 's3' ? S3_ROWS : IPFS_ROWS + return spec.map((r) => ({ ...r, state: 'pending', detail: '' })) +} + +/** Run and show the live test for `profile`; `onDone` gets whether every row passed. */ +export function StorageTest({ profile, onDone }: { profile: StorageProfile; onDone: (ok: boolean) => void }): JSX.Element { + const [rows, setRows] = useState(null) + const [running, setRunning] = useState(false) + const origin = typeof window === 'undefined' ? 'https://forge.dashhq.org' : window.location.origin + + const run = async (): Promise => { + if (running) return + setRunning(true) + setRows(rowsFor(profile)) + const report = (id: RowId, state: RowState, detail: string, cors?: boolean): void => + setRows((prev) => (prev ?? []).map((r) => (r.id === id ? { ...r, state, detail, ...(cors ? { cors } : {}) } : r))) + try { + onDone(await probeProfile(profile, report)) + } catch (e) { + onDone(false) + setRows((prev) => (prev ?? []).map((r) => (r.state === 'running' || r.state === 'pending' ? { ...r, state: 'fail', detail: e instanceof Error ? e.message : String(e) } : r))) + } finally { + setRunning(false) + } + } + + const corsFailed = rows?.some((r) => r.state === 'fail' && r.cors) ?? false + const fix = corsFix(profile.settings.provider, profile.settings.kind === 's3' ? profile.settings.bucket : '', origin) + + return ( +
+
+ + + Runs from this page, so it checks what the browser will do: a few bytes are written and removed again. + +
+ {rows ? ( +
    + {rows.map((r) => { + const m = META[r.state] + return ( +
  1. + + {r.label} + {m.word} + {r.detail} +
  2. + ) + })} +
+ ) : null} + {corsFailed && fix.text ? ( +
+

+ + + Fix CORS. {fix.where} + +

+ +

+ Reads are allowed from any origin (the objects are public); writes only from {origin}. Then press Re-test. +

+
+ ) : null} +
+ ) +} diff --git a/forge-web/components/storage/storage-wizard.tsx b/forge-web/components/storage/storage-wizard.tsx new file mode 100644 index 000000000..3cf9646e8 --- /dev/null +++ b/forge-web/components/storage/storage-wizard.tsx @@ -0,0 +1,370 @@ +'use client' + +/** + * `/settings/storage` — the storage wizard (`ux-dx-spec.md` §3.1): provider tiles, the + * per-provider form, the live browser test, then save; the saved profiles; where browser + * pushes go by default (replication radio); and the cost card, always visible. + * + * Everything saved is sealed in the vault with this browser's key (`lib/storage/store.ts`). + */ + +import { useState } from 'react' +import { Cloud, Database, HardDrive, Link2, Pencil, Server, Trash2, Waypoints } from 'lucide-react' +import { + PROVIDERS, + choiceOf, + policyFor, + policyProblem, + withProfile, + withoutProfile, + type ProviderId, + type ReplicationChoice, + type StorageConfig, + type StorageProfile, +} from '@/lib/storage' +import { useStorageConfig } from '@/hooks/use-storage-config' +import { Button } from '@/components/ui/button' +import { ErrorState, LoadingBlock } from '@/components/ui/states' +import { ProfileForm } from '@/components/storage/profile-form' +import { StorageTest } from '@/components/storage/storage-test' +import { CostCard } from '@/components/storage/cost-card' +import { cn } from '@/lib/utils' +import { formatDate } from '@/lib/view/format' + +const TILE_ICON: Readonly> = { + r2: Cloud, + b2: Database, + aws: Server, + minio: HardDrive, + kubo: Waypoints, + pinning: Link2, + platform: Database, +} + +export function StorageWizard(): JSX.Element { + const { config, loading, error, storable, save, reload } = useStorageConfig() + const [provider, setProvider] = useState(null) + const [editing, setEditing] = useState(null) + + if (loading && !config) return + if (error) return + if (!config) return + + const start = (p: ProviderId, existing: StorageProfile | null = null): void => { + setEditing(existing) + setProvider(p) + } + + return ( +
+ {!storable ? ( +

+ This tab signs with a pasted key, which has no vault. Storage settings are kept encrypted with a stored key: sign in by importing your identity to save them. +

+ ) : null} + +
+

+ Add storage +

+
+ {PROVIDERS.map((p) => { + const Icon = TILE_ICON[p.id] + const active = provider === p.id + return ( + + ) + })} +
+
+ + {provider ? ( + { + setProvider(null) + setEditing(null) + }} + /> + ) : null} + + start(p.settings.provider, p)} /> + + + + +
+ ) +} + +function AddProfile({ + provider, + existing, + config, + storable, + save, + onDone, +}: { + provider: ProviderId + existing: StorageProfile | null + config: StorageConfig + storable: boolean + save: (c: StorageConfig) => Promise + onDone: () => void +}): JSX.Element { + const preset = PROVIDERS.find((p) => p.id === provider) + const [draft, setDraft] = useState<{ profile: StorageProfile; problem: string | null } | null>(null) + const [tested, setTested] = useState(null) + const [saving, setSaving] = useState(false) + const [saveError, setSaveError] = useState(null) + + if (provider === 'platform') { + const has = config.profiles.some((p) => p.settings.kind === 'platform') + return ( +
+

Dash Platform

+

+ Packs stored as Platform documents: permanent, readable by anyone, nothing to run. Priced at about 0.28 DASH per MiB, charged when you push, and never refunded (Platform storage cannot be deleted, so nobody can break a repo others depend on). Every Platform write asks first, with its price. +

+
+ + +
+ {saveError ?

{saveError}

: null} +
+ ) + } + + const nameTaken = draft !== null && existing?.name !== draft.profile.name && config.profiles.some((p) => p.name === draft.profile.name) + const problem = draft?.problem ?? (draft ? null : 'fill in the fields') + const blocking = problem ?? (nameTaken ? 'a profile with that name exists' : null) + + return ( +
+
+

{existing ? `Edit ${existing.name}` : preset?.title}

+ {preset?.blurb} +
+ { + setDraft({ profile, problem: p }) + setTested(null) + }} + /> + {blocking && draft ?

{blocking}

: null} + {draft && !blocking ? : null} +
+ + + + {tested === null ? 'Run the test first.' : tested ? 'All checks passed.' : 'Some checks failed: you can still save, but browser pushes need them all.'} + +
+ {saveError ?

{saveError}

: null} +
+ ) +} + +function Profiles({ config, storable, save, onEdit }: { config: StorageConfig; storable: boolean; save: (c: StorageConfig) => Promise; onEdit: (p: StorageProfile) => void }): JSX.Element { + return ( +
+

+ Your storage +

+ {config.profiles.length === 0 ? ( +

+ No storage yet. Browser pushes (merges, forks) would store packs on Platform at ~0.28 DASH/MiB, asking first. +

+ ) : ( +
    + {config.profiles.map((p) => { + const test = config.lastTests[p.name] + const where = p.settings.kind === 's3' ? `${p.settings.bucket} · ${new URL(p.settings.endpoint).host}` : p.settings.kind === 'platform' ? 'on-chain chunks' : new URL(p.settings.api).host + return ( +
  • + {p.name} + {where} + {test ? ( + + {test.ok ? 'passed' : 'failed some checks'} · {formatDate(test.at)} + + ) : p.settings.kind === 'platform' ? null : ( + not tested + )} + + {p.settings.kind !== 'platform' ? ( + + ) : null} + + +
  • + ) + })} +
+ )} +
+ ) +} + +const CHOICES: readonly { id: ReplicationChoice; label: string }[] = [ + { id: 'one', label: 'One place: the first that confirms is enough' }, + { id: 'all', label: 'Every chosen place: a push fails unless all confirm' }, + { id: 'fallback', label: 'Platform as fallback if my storage fails (costed, asks first)' }, +] + +function DefaultPolicy({ config, storable, save }: { config: StorageConfig; storable: boolean; save: (c: StorageConfig) => Promise }): JSX.Element | null { + const current = config.defaultPolicy + const [targets, setTargets] = useState(() => current?.targets.slice() ?? []) + const [choice, setChoice] = useState(() => (current ? choiceOf(current) : 'one')) + const [saved, setSaved] = useState(false) + const [err, setErr] = useState(null) + if (config.profiles.length === 0) return null + const policy = targets.length > 0 ? policyFor(targets, choice) : null + const problem = policy ? policyProblem(config, policy) : null + + return ( +
+

+ Where browser pushes go +

+

+ The default for every repo you push to from this browser (merges, fork updates, release assets). A repo can override it under its Settings → Storage. The CLI keeps its own policy in git config. +

+
+ Storage to push to + {config.profiles.map((p) => ( + + ))} +
+
+ Replication + {CHOICES.map((c) => ( + + ))} +
+
+ + {problem && targets.length > 0 ? {problem} : null} + {saved ? Saved. : null} + {err ? {err} : null} +
+
+ ) +} diff --git a/forge-web/components/ui/button.tsx b/forge-web/components/ui/button.tsx index 0b4314a44..718b6558f 100644 --- a/forge-web/components/ui/button.tsx +++ b/forge-web/components/ui/button.tsx @@ -25,7 +25,7 @@ const button = cva( ghost: 'bg-transparent text-anvil-700 hover:bg-anvil-100 dark:text-anvil-200 dark:hover:bg-anvil-800', danger: - 'border border-danger/40 bg-transparent text-danger hover:bg-danger/10', + 'border border-danger/40 bg-transparent text-danger-700 hover:bg-danger/10 dark:text-danger-400', subtle: 'bg-anvil-100 text-anvil-800 hover:bg-anvil-200 dark:bg-anvil-800 dark:text-anvil-100 dark:hover:bg-anvil-750', }, diff --git a/forge-web/e2e/helpers.ts b/forge-web/e2e/helpers.ts index 0176d324a..180980aea 100644 --- a/forge-web/e2e/helpers.ts +++ b/forge-web/e2e/helpers.ts @@ -152,7 +152,7 @@ export const PASSPHRASE = 'e2e passphrase for the vault' * per identity instead of registering a new one on every test (identities would otherwise * accumulate keys without bound). Gitignored (e2e/.playwright/). */ -function stateFile(name: string): string { +export function stateFile(name: string): string { return join(__dirname, '.playwright', 'auth', `devnet-${E2E_DEVNET}-${name}.json`) } diff --git a/forge-web/e2e/storage-wizard.spec.ts b/forge-web/e2e/storage-wizard.spec.ts new file mode 100644 index 000000000..2c4f0604c --- /dev/null +++ b/forge-web/e2e/storage-wizard.spec.ts @@ -0,0 +1,133 @@ +import { test, expect, type Page } from '@playwright/test' +import { existsSync } from 'node:fs' +import { E2E_DEVNET, idFile, runAxe, shot, signedIn, stateFile, unlock } from './helpers' + +/** + * The storage wizard (`/settings/storage`) against the local MinIO of infra/docker-compose.yml + * (`forge-byo`: SigV4-signed writes, anonymous reads, CORS for any origin), live on a devnet + * build because a vault needs a signed-in identity: + * + * docker compose -f infra/docker-compose.yml up -d minio minio-init static-http + * E2E_DEVNET=moutai E2E_WRITE=1 pnpm exec playwright test storage-wizard.spec.ts + * + * s1. every row of the browser test passes on MinIO; the profile is saved into the vault, never + * into localStorage; it survives a reload + unlock; a default policy is set. + * s2. a reachable host with no CORS (the static nginx) fails the CORS rows and shows the + * copy-paste fix, prefilled with the bucket and this origin. + * s3. a repo's own policy (its Settings → Your browser pushes). + * s4. axe and a 390 px viewport. + * + * Signs in as MAINTAINER (first run: registers one limited key; later runs reuse the vault). + */ + +test.skip(E2E_DEVNET === '' || process.env['E2E_WRITE'] !== '1', 'needs a devnet build and a signed-in vault: E2E_DEVNET=moutai E2E_WRITE=1') +test.skip(!existsSync(idFile('MAINTAINER')), 'devnet test identities not found') +test.describe.configure({ mode: 'serial', timeout: 240_000 }) + +const MINIO = { endpoint: 'http://127.0.0.1:9000', bucket: 'forge-byo', publicUrl: 'http://127.0.0.1:9000/forge-byo', key: 'minioadmin', secret: 'minioadmin' } +const EMPTY_REPO = { owner: 'GKBTXUdo3MpRYAUqgZvTZGTav9mXGqfJfR5822K2tp79', name: 'forge-v2-empty' } + +async function fillS3(page: Page, v: { name: string; endpoint: string; bucket: string; publicUrl: string }): Promise { + await page.getByLabel('Profile name', { exact: true }).fill(v.name) + await page.getByLabel('S3 endpoint', { exact: true }).fill(v.endpoint) + await page.getByLabel('Region', { exact: true }).fill('us-east-1') + await page.getByLabel('Bucket', { exact: true }).fill(v.bucket) + await page.getByLabel('Public URL', { exact: true }).fill(v.publicUrl) + await page.getByLabel(/^Key prefix/).fill('e2e-web') + await page.getByLabel('Access key id', { exact: true }).fill(MINIO.key) + await page.getByLabel('Secret access key', { exact: true }).fill(MINIO.secret) +} + +test('s1. MinIO passes every browser check, and the profile is sealed in the vault', async ({ browser }) => { + const page = await signedIn(browser, 'MAINTAINER', '/settings/storage/') + // A rerun: remove what an earlier run saved, so this one starts from an empty list. + await expect(page.getByRole('heading', { name: 'Your storage' })).toBeVisible({ timeout: 30_000 }) + page.on('dialog', (d) => void d.accept()) + while ((await page.getByRole('button', { name: /^remove /i }).count()) > 0) { + await page.getByRole('button', { name: /^remove /i }).first().click() + await page.waitForTimeout(500) + } + await shot(page, 'a-01-storage-empty') + await page.getByTestId('tile-minio').click() + await fillS3(page, { name: 'minio-e2e', ...MINIO }) + await page.getByRole('button', { name: /^test$/i }).click() + for (const row of ['put', 'get', 'public', 'cors-range', 'cors-put', 'delete']) { + await expect(page.getByTestId(`probe-${row}`)).toHaveAttribute('data-state', 'ok', { timeout: 60_000 }) + } + await expect(page.getByTestId('cors-fix')).toHaveCount(0) + await shot(page, 'a-02-minio-all-passed') + await page.getByRole('button', { name: /save profile/i }).click() + await expect(page.getByTestId('profile-list')).toContainText('minio-e2e') + await expect(page.getByTestId('profile-list')).toContainText('passed') + + // Default policy: this profile, one copy. + await page.getByRole('checkbox', { name: 'minio-e2e' }).first().check() + await page.getByRole('radio', { name: /one place/i }).check() + await page.getByRole('button', { name: /save default/i }).click() + await expect(page.getByText('Saved.')).toBeVisible() + // Later tests restore this context's IndexedDB: keep the (sealed) settings in it. + await page.context().storageState({ path: stateFile('MAINTAINER'), indexedDB: true }) + + // Secrets never reach localStorage; IndexedDB holds only ciphertext. + const stores = await page.evaluate(async () => { + const ls = JSON.stringify(Object.entries(localStorage)) + const idb = await new Promise((resolve) => { + const req = indexedDB.open('dash-forge') + req.onsuccess = () => { + const tx = req.result.transaction('vault', 'readonly') + const all = tx.objectStore('vault').getAll() + all.onsuccess = () => resolve(JSON.stringify(all.result, (_k, v: unknown) => (v instanceof Uint8Array ? `<${v.length} bytes>` : v))) + } + }) + return { ls, idb } + }) + for (const s of [stores.ls, stores.idb]) { + expect(s).not.toContain('minioadmin') + expect(s).not.toContain('forge-byo') + } + + // A reload locks the vault; the settings come back after unlocking. + await page.reload({ waitUntil: 'domcontentloaded' }) + await unlock(page) + await expect(page.getByTestId('profile-list')).toContainText('minio-e2e', { timeout: 30_000 }) + await shot(page, 'a-03-profile-saved') +}) + +test('s2. a reachable host without CORS shows the copy-paste fix', async ({ browser }) => { + const page = await signedIn(browser, 'MAINTAINER', '/settings/storage/') + await page.getByTestId('tile-r2').click() + await fillS3(page, { name: 'no-cors', endpoint: 'http://127.0.0.1:8082', bucket: 'my-bucket', publicUrl: 'http://127.0.0.1:8082/my-bucket' }) + await page.getByRole('button', { name: /^test$/i }).click() + await expect(page.getByTestId('probe-cors-put')).toHaveAttribute('data-state', 'fail', { timeout: 60_000 }) + const fix = page.getByTestId('cors-fix') + await expect(fix).toBeVisible() + await expect(fix).toContainText('R2 → my-bucket → Settings → CORS Policy') + await expect(fix).toContainText('"AllowedMethods"') + await expect(fix).toContainText(new URL(page.url()).origin) + await expect(fix).toContainText('x-amz-content-sha256') + await shot(page, 'a-04-cors-fix') + await page.getByRole('button', { name: /cancel/i }).click() +}) + +test('s3. a repo can override where browser pushes go', async ({ browser }) => { + const page = await signedIn(browser, 'MAINTAINER', `/repo/settings/?owner=${EMPTY_REPO.owner}&name=${EMPTY_REPO.name}`) + const box = page.getByTestId('repo-storage-policy') + await expect(box).toContainText('Using your default', { timeout: 60_000 }) + await box.getByRole('combobox').selectOption('fallback') + await box.getByRole('button', { name: /use for this repo/i }).click() + await expect(box).toContainText('Saved for this repo.') + await expect(box).toContainText('This repo has its own choice.') + await shot(page, 'a-05-repo-policy') + await box.getByRole('button', { name: /use my default/i }).click() + await expect(box).toContainText('Back to your default.') +}) + +test('s4. accessible, and usable at 390 px', async ({ browser }) => { + const page = await signedIn(browser, 'MAINTAINER', '/settings/storage/') + await page.getByTestId('tile-r2').click() + expect(await runAxe(page, 'storage wizard')).toEqual([]) + await page.setViewportSize({ width: 390, height: 844 }) + await expect(page.getByText(/use a desktop browser for this step/i)).toBeVisible() + await expect(page.getByTestId('tile-minio')).toBeVisible() + await shot(page, 'a-06-mobile') +}) diff --git a/forge-web/hooks/use-storage-config.ts b/forge-web/hooks/use-storage-config.ts new file mode 100644 index 000000000..7e877d57e --- /dev/null +++ b/forge-web/hooks/use-storage-config.ts @@ -0,0 +1,45 @@ +'use client' + +/** + * useStorageConfig — the signed-in identity's storage configuration, read from and written to + * the vault. Null while signed out or locked (the settings are sealed with the key; there is + * nothing to show without it). A tab-only pasted key has no vault record, so it cannot store + * settings; `storable` says so. + */ + +import { useCallback } from 'react' + +import { useAuth } from '@/contexts/auth-context' +import { useAsync } from '@/hooks/use-async' +import { DEFAULT_NETWORK } from '@/lib/constants' +import { EMPTY_STORAGE_CONFIG, loadStorageConfig, saveStorageConfig, type StorageConfig } from '@/lib/storage' + +export interface StorageConfigState { + readonly config: StorageConfig | null + readonly loading: boolean + readonly error: string | null + /** Whether this session can store settings (a vault-stored key). */ + readonly storable: boolean + save: (next: StorageConfig) => Promise + reload: () => void +} + +export function useStorageConfig(): StorageConfigState { + const { identity, storage } = useAuth() + const storable = identity !== null && storage === 'vault' + const state = useAsync( + () => (storable ? loadStorageConfig(DEFAULT_NETWORK, identity) : Promise.resolve(EMPTY_STORAGE_CONFIG)), + [identity ?? '', storage ?? ''], + { enabled: identity !== null }, + ) + const { reload } = state + const save = useCallback( + async (next: StorageConfig): Promise => { + if (!storable || identity === null) throw new Error('Storage settings are kept in the vault: sign in with a stored key (not a pasted one).') + await saveStorageConfig(DEFAULT_NETWORK, identity, next) + reload() + }, + [identity, reload, storable], + ) + return { config: identity === null ? null : state.data, loading: state.loading, error: state.error, storable, save, reload } +} diff --git a/forge-web/lib/auth/vault.ts b/forge-web/lib/auth/vault.ts index 64c1fc018..c533be5af 100644 --- a/forge-web/lib/auth/vault.ts +++ b/forge-web/lib/auth/vault.ts @@ -14,6 +14,11 @@ * record cannot be replayed under another identity. * - In memory: the unlocked record lives only in this module (never React state, never * localStorage), for at most {@link AUTO_LOCK_MS} (12 h) or until {@link lockVault}. + * - Storage credentials (`ux-dx-spec.md` §3.1: bucket keys, pinning tokens) are a second + * AES-GCM blob beside the record (`vault-storage::`), sealed under a key + * HKDF-derived from the same data key. While unlocked, only that derived key is held, as a + * non-extractable `CryptoKey`; it is dropped with the rest on lock. A renewal that replaces + * the record re-seals the blob under the new data key. * * Per-origin by construction: IndexedDB and WebAuthn (`rpId` = this host) are origin-scoped. */ @@ -112,6 +117,53 @@ function aad(network: Network, identityId: string, slot: string): Uint8Array { return enc.encode(`dash-forge vault v1|${network}|${identityId}|${slot}`) } +function storageBlobKey(network: Network, identityId: string): string { + return `vault-storage:${network}:${identityId}` +} + +/** The sealed storage-credentials blob. */ +interface StorageBlob { + readonly iv: Uint8Array + readonly ciphertext: Uint8Array +} + +/** The storage-blob key: HKDF(data key) → a non-extractable AES-GCM key. */ +async function deriveStorageKey(dataKey: Uint8Array, network: Network, identityId: string): Promise { + const raw = hkdf(sha256, dataKey, enc.encode('dash-forge vault storage v1'), enc.encode(`${network}|${identityId}`), 32) + try { + return await crypto.subtle.importKey('raw', buf(raw), { name: 'AES-GCM' }, false, ['encrypt', 'decrypt']) + } finally { + raw.fill(0) + } +} + +async function sealBlob(key: CryptoKey, network: Network, identityId: string, value: unknown): Promise { + const iv = random(12) + const plain = enc.encode(JSON.stringify(value)) + try { + const ct = await crypto.subtle.encrypt({ name: 'AES-GCM', iv: buf(iv), additionalData: buf(aad(network, identityId, 'storage')) }, key, buf(plain)) + return { iv, ciphertext: new Uint8Array(ct) } + } finally { + plain.fill(0) + } +} + +async function openBlob(key: CryptoKey, network: Network, identityId: string, blob: StorageBlob): Promise { + let plain: Uint8Array + try { + plain = new Uint8Array( + await crypto.subtle.decrypt({ name: 'AES-GCM', iv: buf(blob.iv), additionalData: buf(aad(network, identityId, 'storage')) }, key, buf(blob.ciphertext)), + ) + } catch { + throw new VaultLockedError('the stored storage settings do not open with this key') + } + try { + return JSON.parse(new TextDecoder().decode(plain)) as unknown + } finally { + plain.fill(0) + } +} + function random(n: number): Uint8Array { return crypto.getRandomValues(new Uint8Array(n)) } @@ -248,7 +300,11 @@ export async function storeInVault(network: Network, secret: VaultSecret, protec throw new Error(`use a passphrase of at least ${MIN_PASSPHRASE} characters`) } const { identityId } = secret + // A renewal replaces the record (and its data key): carry the storage settings across when + // this session can open them; otherwise they cannot be opened any more and are dropped. + const carried = await readStorageBlob(network, identityId).catch(() => null) const dataKey = random(32) + let storageKey: CryptoKey try { const body = await seal(dataKey, enc.encode(JSON.stringify(secret)), aad(network, identityId, 'body')) const slots: Slot[] = [] @@ -275,11 +331,40 @@ export async function storeInVault(network: Network, secret: VaultSecret, protec ciphertext: body.ct, slots, } + storageKey = await deriveStorageKey(dataKey, network, identityId) await idbPut('vault', key(network, identityId), record) + if (carried !== null) await idbPut('vault', storageBlobKey(network, identityId), await sealBlob(storageKey, network, identityId, carried)) + else await idbDelete('vault', storageBlobKey(network, identityId)) } finally { dataKey.fill(0) } - setUnlocked(network, secret) + setUnlocked(network, secret, storageKey) +} + +/** + * The storage settings sealed for (network, identity), or null when none are stored. Needs the + * vault unlocked for that identity ({@link VaultLockedError} otherwise). The caller parses the + * value (it is the caller's schema). + */ +export async function readStorageBlob(network: Network, identityId: string): Promise { + const k = unlockedStorageKey(network, identityId) + const blob = await idbGet('vault', storageBlobKey(network, identityId)) + if (blob === undefined) return null + if (k === null) throw new VaultLockedError('unlock to read your storage settings') + return openBlob(k, network, identityId, blob) +} + +/** Seal `value` as the storage settings of (network, identity). Needs the vault unlocked. */ +export async function writeStorageBlob(network: Network, identityId: string, value: unknown): Promise { + assertDedicatedOrigin() + const k = unlockedStorageKey(network, identityId) + if (k === null) throw new VaultLockedError('unlock with a stored key to save storage settings') + await idbPut('vault', storageBlobKey(network, identityId), await sealBlob(k, network, identityId, value)) +} + +function unlockedStorageKey(network: Network, identityId: string): CryptoKey | null { + if (unlockedSecret(network, identityId) === null) return null + return unlocked?.storageKey ?? null } /** The vaults stored for `network` (no secrets). */ @@ -293,14 +378,14 @@ export async function listVaults(network: Network): Promise { })) } -async function unwrapWith(network: Network, record: VaultRecord, kek: Uint8Array, slot: Slot): Promise { +async function unwrapWith(network: Network, record: VaultRecord, kek: Uint8Array, slot: Slot): Promise<{ secret: VaultSecret; storageKey: CryptoKey }> { const dataKey = await open(kek, slot.iv, slot.wrapped, aad(network, record.identityId, slot.kind)) try { const body = await open(dataKey, record.iv, record.ciphertext, aad(network, record.identityId, 'body')) const secret = JSON.parse(new TextDecoder().decode(body)) as VaultSecret body.fill(0) if (secret.identityId !== record.identityId) throw new VaultLockedError('vault record does not match its identity') - return secret + return { secret, storageKey: await deriveStorageKey(dataKey, network, record.identityId) } } finally { dataKey.fill(0) kek.fill(0) @@ -316,8 +401,8 @@ export async function unlockWithPassphrase(network: Network, identityId: string, // The parameters are pinned, never read from the (unauthenticated) record: a tampered // record cannot make unlock allocate gigabytes. if (JSON.stringify(slot.params) !== JSON.stringify(ARGON2_PARAMS)) throw new VaultLockedError('this key was stored with unsupported settings') - const secret = await unwrapWith(network, record, await passphraseKey(passphrase, slot.salt), slot) - setUnlocked(network, secret) + const { secret, storageKey } = await unwrapWith(network, record, await passphraseKey(passphrase, slot.salt), slot) + setUnlocked(network, secret, storageKey) return secret } @@ -332,8 +417,8 @@ export async function unlockWithPasskey(network: Network, identityId: string): P const raw = new Uint8Array(output) const kek = prfKey(raw, network, identityId) raw.fill(0) - const secret = await unwrapWith(network, record, kek, slot) - setUnlocked(network, secret) + const { secret, storageKey } = await unwrapWith(network, record, kek, slot) + setUnlocked(network, secret, storageKey) return secret } @@ -341,6 +426,7 @@ export async function unlockWithPasskey(network: Network, identityId: string): P export async function forgetVault(network: Network, identityId: string): Promise { lockVault() await idbDelete('vault', key(network, identityId)) + await idbDelete('vault', storageBlobKey(network, identityId)) clearSignedWrites() } @@ -359,8 +445,8 @@ export function clearSignedWrites(): void { } } -// The unlocked secret: module memory only. -let unlocked: { network: Network; secret: VaultSecret; at: number } | null = null +// The unlocked secret (and the storage-blob key, for a vault-stored key): module memory only. +let unlocked: { network: Network; secret: VaultSecret; at: number; storageKey: CryptoKey | null } | null = null let lockTimer: ReturnType | null = null const lockListeners = new Set<() => void>() @@ -372,8 +458,8 @@ export function onVaultLock(listener: () => void): () => void { } } -function setUnlocked(network: Network, secret: VaultSecret): void { - unlocked = { network, secret, at: Date.now() } +function setUnlocked(network: Network, secret: VaultSecret, storageKey: CryptoKey | null = null): void { + unlocked = { network, secret, at: Date.now(), storageKey } if (lockTimer) clearTimeout(lockTimer) lockTimer = setTimeout(lockVault, AUTO_LOCK_MS) } diff --git a/forge-web/lib/design/contrast.test.ts b/forge-web/lib/design/contrast.test.ts index 79725721c..3d04149bd 100644 --- a/forge-web/lib/design/contrast.test.ts +++ b/forge-web/lib/design/contrast.test.ts @@ -102,6 +102,21 @@ describe('Verification state words meet WCAG AA', () => { }) }) +describe('danger text meets WCAG AA', () => { + const danger = colors['danger'] as Record + it.each(Object.entries(LIGHT_SURFACES))('danger-700 on %s (light theme)', (_, bg) => { + expect(contrast(rgb(danger['700']!), rgb(bg!))).toBeGreaterThanOrEqual(AA_TEXT) + }) + it.each(Object.entries(DARK_SURFACES))('danger-400 on %s (dark theme)', (_, bg) => { + expect(contrast(rgb(danger['400']!), rgb(bg!))).toBeGreaterThanOrEqual(AA_TEXT) + }) + it('the base value is not a text color on the darkest or lightest-gray surface', () => { + // 4.01:1 on anvil-950: the storage wizard's axe failure (a danger button). + expect(contrast(rgb(danger.DEFAULT!), rgb(anvil['950']!))).toBeLessThan(AA_TEXT) + expect(contrast(rgb(danger.DEFAULT!), rgb(anvil['100']!))).toBeLessThan(AA_TEXT) + }) +}) + describe('no component renders text in the raw brand blue', () => { const root = resolve(__dirname, '../..') diff --git a/forge-web/lib/repo/push.ts b/forge-web/lib/repo/push.ts new file mode 100644 index 000000000..7798113fa --- /dev/null +++ b/forge-web/lib/repo/push.ts @@ -0,0 +1,203 @@ +/** + * Push-side chain writes for forge-v2 repos: `packManifest` and Platform `chunk` documents, + * shaped exactly as forge-core writes them (`RepoService::write_pack_manifest`, + * `backends/platform.rs::encode_chunk_doc`, `pack::split`), so a pack the browser records is + * read by the CLI and vice versa. + * + * Every write is member-gated at consensus (`ownerRefersTo` maintainer or writer) and + * idempotent here: the unique indexes are `(repoId, $ownerId, packHash)` for a manifest and + * `(repoId, $ownerId, packHash, seq)` for a chunk, so each checks the signer's own copy first + * and a duplicate-unique refusal (a lost answer to an earlier attempt) counts as done. + */ + +import type { EvoSDK } from '@dashevo/evo-sdk' +import { hexToBytes } from '@noble/hashes/utils.js' + +import { CHUNK_FIELDS, FIELD_MAX, MANIFEST_MAX_URIS, MANIFEST_URI_MAX_LEN } from '../constants' +import { decodeIdentifier } from '../auth/base58' +import { + ConsensusRefusal, + DUPLICATE_UNIQUE_CODE, + createDocumentIdempotent, + hexToBase64, + previewCredits, + queryDocumentsWithProof, + type WriteAuth, + type WriteResult, +} from '../sdk' +import { DOC, type V2RepoRef } from './contract' +import { repoSource } from './source' + +/** The fields of a `packManifest` (forge-core `PackManifestInput`). */ +export interface PackManifestInput { + /** Hex SHA-256 of the artifact. */ + readonly packHash: string + /** 0 git pack | 1 objectLocator | 2 flatIndex. */ + readonly kind: number + readonly sizeBytes: number + readonly objectCount: number + /** Platform chunk documents holding a copy (0 when none). */ + readonly chunkCount: number + /** 0 = chunks on Platform, 1 = external only. */ + readonly storage: 0 | 1 + /** Where the bytes are: a `platform://` locator first when chunks exist, then public URLs. */ + readonly uris: readonly string[] + /** flatIndex only: the tip commit oids it indexes (hex). */ + readonly tips?: readonly string[] + /** Hex pack hashes this one supersedes. */ + readonly supersedes?: readonly string[] +} + +/** Why `uris` does not fit the manifest's typed array, or null (`uris` ≤ 8 × ≤ 300 bytes). */ +export function manifestUrisProblem(uris: readonly string[]): string | null { + if (uris.length === 0) return 'no confirmed copy recorded any URI; refusing to write a manifest nothing can read' + if (uris.length > MANIFEST_MAX_URIS) return `a manifest holds at most ${MANIFEST_MAX_URIS} URIs` + if (uris.some((u) => new TextEncoder().encode(u).length > MANIFEST_URI_MAX_LEN)) return `a manifest URI holds at most ${MANIFEST_URI_MAX_LEN} bytes` + return null +} + +function concatHex(list: readonly string[], width: number): Uint8Array { + const out = new Uint8Array(list.length * width) + list.forEach((h, i) => { + const b = hexToBytes(h) + if (b.length !== width) throw new Error(`expected ${width}-byte entries, got ${b.length}`) + out.set(b, i * width) + }) + return out +} + +function isDuplicate(e: unknown): boolean { + return e instanceof ConsensusRefusal && e.code === DUPLICATE_UNIQUE_CODE +} + +/** The signer's own manifest of `packHash` in `repo` (the unique slot), or null. */ +export async function findOwnManifest(sdk: EvoSDK, repo: V2RepoRef, ownerId: string, packHashHex: string): Promise { + const { documents } = await queryDocumentsWithProof( + sdk, + repoSource(repo).repoQuery(DOC.packManifest, { + where: [ + ['$ownerId', '==', ownerId], + ['packHash', '==', hexToBase64(packHashHex)], + ], + limit: 1, + }), + ) + const id = documents[0]?.['$id'] + return typeof id === 'string' ? id : null +} + +/** + * Write a `packManifest` (member-gated). Returns the signer's manifest id: an existing one + * (nothing spent) when this signer already recorded the pack. + */ +export async function writePackManifest( + sdk: EvoSDK, + auth: WriteAuth, + repo: V2RepoRef, + input: PackManifestInput, + intent?: string, +): Promise { + const problem = manifestUrisProblem(input.uris) + if (problem) throw new Error(problem) + const existing = await findOwnManifest(sdk, repo, auth.identityId, input.packHash) + if (existing !== null) return { documentId: existing, confirmed: true, cost: previewCredits(0), actualCredits: 0 } + const data: Record = { + repoId: decodeIdentifier(repo.repoId), + packHash: hexToBytes(input.packHash), + kind: input.kind, + sizeBytes: input.sizeBytes, + objectCount: input.objectCount, + chunkCount: input.chunkCount, + storage: input.storage, + offsetIndexParts: 0, + uris: [...input.uris], + } + if (input.tips && input.tips.length > 0) data['tips'] = concatHex(input.tips, 20) + if (input.supersedes && input.supersedes.length > 0) data['supersedes'] = concatHex(input.supersedes, 32) + try { + return await createDocumentIdempotent(sdk, auth, { contractId: repo.forge.core, documentType: DOC.packManifest, data, ...(intent ? { intent } : {}) }) + } catch (e) { + if (!isDuplicate(e)) throw e + const id = await findOwnManifest(sdk, repo, auth.identityId, input.packHash) + if (id === null) throw e + return { documentId: id, confirmed: true, cost: previewCredits(0), actualCredits: 0 } + } +} + +/** One chunk: its seq and up to three ≤ 4,900-byte fields (forge-core `pack::split`). */ +export interface Chunk { + readonly seq: number + readonly fields: readonly Uint8Array[] +} + +/** Split `bytes` into chunks exactly as forge-core `pack::split` does. */ +export function splitChunks(bytes: Uint8Array): Chunk[] { + const chunks: Chunk[] = [] + let fields: Uint8Array[] = [] + for (let at = 0; at < bytes.length; at += FIELD_MAX) { + fields.push(bytes.subarray(at, at + FIELD_MAX)) + if (fields.length === CHUNK_FIELDS) { + chunks.push({ seq: chunks.length, fields }) + fields = [] + } + } + if (fields.length > 0) chunks.push({ seq: chunks.length, fields }) + return chunks +} + +/** The seqs of `packHash` the signer already stored in `repo`. */ +async function storedSeqs(sdk: EvoSDK, repo: V2RepoRef, ownerId: string, packHashHex: string, total: number): Promise> { + const have = new Set() + const source = repoSource(repo) + for (let start = 0; start < total; start += 100) { + const seqs = Array.from({ length: Math.min(100, total - start) }, (_, i) => start + i) + const { documents } = await queryDocumentsWithProof(sdk, source.chunkQuery(packHashHex, ownerId, seqs)) + for (const d of documents) { + const raw = d['seq'] + const n = typeof raw === 'bigint' ? Number(raw) : typeof raw === 'number' ? raw : -1 + if (n >= 0) have.add(n) + } + } + return have +} + +/** + * Store `bytes` as the signer's `chunk` documents in `repo` (resumable: seqs already on chain + * are skipped, so a retry after an interruption pays only for what is missing). Returns the + * `platform://///` locator a manifest records. + */ +export async function putPlatformChunks( + sdk: EvoSDK, + auth: WriteAuth, + repo: V2RepoRef, + bytes: Uint8Array, + packHashHex: string, + onProgress?: (done: number, total: number) => void, +): Promise<{ locator: string; chunkCount: number }> { + const chunks = splitChunks(bytes) + const have = await storedSeqs(sdk, repo, auth.identityId, packHashHex, chunks.length) + const R = decodeIdentifier(repo.repoId) + const hash = hexToBytes(packHashHex) + let done = have.size + onProgress?.(done, chunks.length) + for (const chunk of chunks) { + if (have.has(chunk.seq)) continue + const data: Record = { repoId: R, packHash: hash, seq: chunk.seq } + chunk.fields.forEach((f, i) => { + data[`d${i}`] = new Uint8Array(f) + }) + try { + await createDocumentIdempotent(sdk, auth, { + contractId: repo.forge.core, + documentType: DOC.chunk, + data, + intent: `chunk:${repo.repoId}:${packHashHex}:${chunk.seq}`, + }) + } catch (e) { + if (!isDuplicate(e)) throw e + } + done += 1 + onProgress?.(done, chunks.length) + } + return { locator: `platform://${repo.forge.core}/${repo.repoId}/${auth.identityId}/${packHashHex}`, chunkCount: chunks.length } +} diff --git a/forge-web/lib/sdk/cost.ts b/forge-web/lib/sdk/cost.ts index 6ba8fae17..ead306e6d 100644 --- a/forge-web/lib/sdk/cost.ts +++ b/forge-web/lib/sdk/cost.ts @@ -34,6 +34,10 @@ export const BASE_CREDITS: Readonly> = { label: 45_000_000, refUpdate: 45_000_000, protectedRefUpdate: 45_000_000, + // Not yet measured from the browser: forge-core's per-byte model (fixed shape plus the + // storage of its byte fields, which `textBytes` does not count — see `estimateBytesCredits`). + packManifest: 60_000_000, + chunk: 20_000_000, // forge-collab issue: 57_400_000, patch: 112_000_000, @@ -119,6 +123,21 @@ export function previewCreate(documentType: string, data: Readonly> = {}): number { + return estimateCreateCredits(documentType, data) + CREDITS_PER_TEXT_BYTE * byteLen +} + +/** + * Estimated credits to store `bytes` as Platform `chunk` documents (three 4,900-byte fields + * each, forge-core `pack::split`): every chunk's fixed cost plus the storage of its bytes. + * Roughly 0.28 DASH per MiB — Platform storage is permanent. + */ +export function estimateChunkCredits(bytes: number): number { + const chunks = Math.max(1, Math.ceil(bytes / (4900 * 3))) + return chunks * (BASE_CREDITS['chunk'] ?? DEFAULT_BASE_CREDITS) + CREDITS_PER_TEXT_BYTE * bytes +} + /** Preview for deleting one document of `documentType` (usually a refund: negative credits). */ export function previewDelete(documentType: string): CostPreview { return previewCredits(DELETE_CREDITS[documentType] ?? DEFAULT_DELETE_CREDITS) diff --git a/forge-web/lib/storage/cid.test.ts b/forge-web/lib/storage/cid.test.ts new file mode 100644 index 000000000..693e18490 --- /dev/null +++ b/forge-web/lib/storage/cid.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest' + +import { CID_CHUNK_SIZE, cidV1RawLeaves, isCid } from './cid' + +describe('cidV1RawLeaves (parity with forge-core backends/cid.rs)', () => { + it('derives the well-known empty raw block', () => { + expect(cidV1RawLeaves(new Uint8Array(0))).toBe('bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku') + }) + + it('derives a single raw leaf', () => { + expect(cidV1RawLeaves(new TextEncoder().encode('hello'))).toBe('bafkreibm6jg3ux5qumhcn2b3flc3tyu6dmlb4xa7u5bf44yegnrjhc4yeq') + }) + + it('makes a dag-pb root past one chunk, deterministically', () => { + const data = new Uint8Array(CID_CHUNK_SIZE + 1).fill(7) + const cid = cidV1RawLeaves(data) + expect(cid.startsWith('bafybei')).toBe(true) + expect(cidV1RawLeaves(data)).toBe(cid) + // What `ipfs add --only-hash` returns for these bytes under the pinned parameters. + expect(cid).toBe('bafybeihldj5jykexbnvf5jx3cm2pjlqqa3hwyc5d6wo2y7ujlk7qjjgzlu') + }) + + it('recognizes CIDs and refuses path smuggling', () => { + expect(isCid('bafkreibm6jg3ux5qumhcn2b3flc3tyu6dmlb4xa7u5bf44yegnrjhc4yeq')).toBe(true) + expect(isCid('bafy/../../api')).toBe(false) + expect(isCid('')).toBe(false) + }) +}) diff --git a/forge-web/lib/storage/cid.ts b/forge-web/lib/storage/cid.ts new file mode 100644 index 000000000..a653099d4 --- /dev/null +++ b/forge-web/lib/storage/cid.ts @@ -0,0 +1,121 @@ +/** + * Local CIDv1 derivation for the exact import parameters Forge asks kubo for + * (`cid-version=1&raw-leaves=true&chunker=size-262144&hash=sha2-256`, balanced layout, 174 + * links per node). A port of forge-core `backends/cid.rs`: the CID kubo returns must equal this + * re-derivation, so the CID a manifest records is a second integrity check next to its sha256. + */ + +import { sha256 } from '@noble/hashes/sha2.js' + +/** kubo's fixed-size chunker (`size-262144`). */ +export const CID_CHUNK_SIZE = 262_144 +/** Links per UnixFS node in the balanced layout (go-unixfs `DefaultLinksPerBlock`). */ +export const CID_MAX_LINKS = 174 + +const CODEC_RAW = 0x55 +const CODEC_DAG_PB = 0x70 +const MH_SHA2_256 = 0x12 + +interface Block { + readonly cid: Uint8Array + readonly tsize: number + readonly content: number +} + +function varint(out: number[], v: number): void { + let n = v + for (;;) { + const byte = n % 128 + n = Math.floor(n / 128) + if (n === 0) { + out.push(byte) + return + } + out.push(byte | 0x80) + } +} + +function key(out: number[], field: number, wire: number): void { + varint(out, field * 8 + wire) +} + +function cidBytes(codec: number, block: Uint8Array): Uint8Array { + const out: number[] = [] + varint(out, 1) + varint(out, codec) + varint(out, MH_SHA2_256) + varint(out, 32) + return Uint8Array.from([...out, ...sha256(block)]) +} + +function leaf(chunk: Uint8Array): Block { + return { cid: cidBytes(CODEC_RAW, chunk), tsize: chunk.length, content: chunk.length } +} + +/** A dag-pb UnixFS `File` node linking `children` (links before data, the canonical order). */ +function fileNode(children: readonly Block[]): Block { + const content = children.reduce((s, c) => s + c.content, 0) + const unixfs: number[] = [] + key(unixfs, 1, 0) + varint(unixfs, 2) + key(unixfs, 3, 0) + varint(unixfs, content) + for (const c of children) { + key(unixfs, 4, 0) + varint(unixfs, c.content) + } + const node: number[] = [] + for (const c of children) { + const link: number[] = [] + key(link, 1, 2) + varint(link, c.cid.length) + link.push(...c.cid) + key(link, 2, 2) + varint(link, 0) + key(link, 3, 0) + varint(link, c.tsize) + key(node, 2, 2) + varint(node, link.length) + node.push(...link) + } + key(node, 1, 2) + varint(node, unixfs.length) + node.push(...unixfs) + const bytes = Uint8Array.from(node) + return { cid: cidBytes(CODEC_DAG_PB, bytes), tsize: bytes.length + children.reduce((s, c) => s + c.tsize, 0), content } +} + +function base32(bytes: Uint8Array): string { + const alphabet = 'abcdefghijklmnopqrstuvwxyz234567' + let out = 'b' + let buffer = 0 + let bits = 0 + for (const b of bytes) { + buffer = ((buffer << 8) | b) & 0xffff + bits += 8 + while (bits >= 5) { + bits -= 5 + out += alphabet[(buffer >> bits) & 31] + } + } + if (bits > 0) out += alphabet[(buffer << (5 - bits)) & 31] + return out +} + +/** The CIDv1 (base32, `b…`) kubo produces for `bytes` under Forge's pinned import parameters. */ +export function cidV1RawLeaves(bytes: Uint8Array): string { + let level: Block[] = [] + if (bytes.length === 0) level.push(leaf(bytes)) + for (let i = 0; i < bytes.length; i += CID_CHUNK_SIZE) level.push(leaf(bytes.subarray(i, i + CID_CHUNK_SIZE))) + while (level.length > 1) { + const next: Block[] = [] + for (let i = 0; i < level.length; i += CID_MAX_LINKS) next.push(fileNode(level.slice(i, i + CID_MAX_LINKS))) + level = next + } + return base32((level[0] as Block).cid) +} + +/** Whether `s` is a plausible CID (base-encoded: letters and digits only). */ +export function isCid(s: string): boolean { + return /^[A-Za-z0-9]+$/.test(s) +} diff --git a/forge-web/lib/storage/cors.ts b/forge-web/lib/storage/cors.ts new file mode 100644 index 000000000..70be53c7b --- /dev/null +++ b/forge-web/lib/storage/cors.ts @@ -0,0 +1,83 @@ +/** + * The exact CORS configuration to paste for each provider (`ux-dx-spec.md` §3.1 step 3), + * prefilled with the bucket and this app's origin. + * + * Two rules, as in forge-core `storage/cors.rs` plus the write half the browser needs: + * - **read**, any origin: `GET`/`HEAD` with `Range`, exposing `Content-Range`, `Content-Length` + * and `ETag`. The objects are public; every Forge web app (and any mirror of it) reads them. + * - **write**, this app's origin only: `PUT`/`GET`/`HEAD`/`DELETE` with the SigV4 headers, so + * browser pushes and merges can upload. Listed header by header rather than `x-amz-*`, + * because not every provider accepts a wildcard there. + */ + +import type { ProviderId } from './profiles' + +/** The headers a signed browser request sends that are not CORS-safelisted. */ +export const SIGNED_HEADERS = ['authorization', 'content-type', 'range', 'x-amz-content-sha256', 'x-amz-date', 'x-amz-security-token'] as const +const EXPOSE = ['Content-Range', 'Content-Length', 'ETag'] as const + +/** A fix: where to apply it, and the text to paste or run. */ +export interface CorsFix { + readonly where: string + readonly text: string +} + +function s3Rules(origin: string): unknown[] { + return [ + { AllowedOrigins: ['*'], AllowedMethods: ['GET', 'HEAD'], AllowedHeaders: ['Range'], ExposeHeaders: [...EXPOSE], MaxAgeSeconds: 86400 }, + { AllowedOrigins: [origin], AllowedMethods: ['PUT', 'GET', 'HEAD', 'DELETE'], AllowedHeaders: [...SIGNED_HEADERS], ExposeHeaders: [...EXPOSE], MaxAgeSeconds: 86400 }, + ] +} + +/** The CORS fix for `provider`, prefilled with `bucket` and the app `origin`. */ +export function corsFix(provider: ProviderId, bucket: string, origin: string): CorsFix { + const b = bucket || '' + switch (provider) { + case 'r2': + return { + where: `Cloudflare dashboard → R2 → ${b} → Settings → CORS Policy → Add CORS policy. Paste:`, + text: JSON.stringify(s3Rules(origin), null, 2), + } + case 'aws': + return { + where: `Save as cors.json, then run: aws s3api put-bucket-cors --bucket ${b} --cors-configuration file://cors.json`, + text: JSON.stringify({ CORSRules: s3Rules(origin) }, null, 2), + } + case 'b2': + return { + where: `Save as cors.json, then run: b2 bucket update --cors-rules "$(cat cors.json)" ${b} allPublic`, + text: JSON.stringify( + [ + { corsRuleName: 'dashForgeRead', allowedOrigins: ['*'], allowedOperations: ['s3_get', 's3_head', 'b2_download_file_by_name'], allowedHeaders: ['range'], exposeHeaders: EXPOSE.map((h) => h.toLowerCase()), maxAgeSeconds: 86400 }, + { corsRuleName: 'dashForgeWrite', allowedOrigins: [origin], allowedOperations: ['s3_put', 's3_get', 's3_head', 's3_delete'], allowedHeaders: [...SIGNED_HEADERS], exposeHeaders: EXPOSE.map((h) => h.toLowerCase()), maxAgeSeconds: 86400 }, + ], + null, + 2, + ), + } + case 'minio': + return { + where: 'MinIO answers CORS for every origin by default. If it was restricted, allow this app, then make the bucket publicly readable (not writable):', + text: [`mc admin config set api cors_allow_origin="${origin}"`, 'mc admin service restart ', `mc anonymous set download /${b}`].join('\n'), + } + case 'kubo': + case 'pinning': + return { + where: 'kubo refuses RPC calls from web pages unless their origin is allowed. Run, then restart the daemon:', + text: kuboCorsLines(origin), + } + case 'platform': + return { where: 'Dash Platform needs no CORS setup.', text: '' } + } +} + +/** `ipfs config` lines allowing this app to call the RPC API and read the gateway. */ +export function kuboCorsLines(origin: string): string { + return [ + `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin '["${origin}"]'`, + `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Methods '["POST"]'`, + `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Headers '["Authorization"]'`, + `ipfs config --json Gateway.HTTPHeaders.Access-Control-Allow-Origin '["*"]'`, + `ipfs config --json Gateway.HTTPHeaders.Access-Control-Expose-Headers '["Content-Range", "Content-Length", "ETag"]'`, + ].join('\n') +} diff --git a/forge-web/lib/storage/index.ts b/forge-web/lib/storage/index.ts new file mode 100644 index 000000000..cb19546a3 --- /dev/null +++ b/forge-web/lib/storage/index.ts @@ -0,0 +1,76 @@ +/** + * Bring-your-own storage in the browser (`ux-dx-spec.md` §3.1): profiles sealed in the vault, + * the live bucket test, and the upload path browser writes that push packs (merge, fork + * follow-ups, release assets) go through. + */ + +import type { EvoSDK } from '@dashevo/evo-sdk' + +import type { WriteAuth, WriteResult } from '../sdk' +import type { V2RepoRef } from '../repo/contract' +import { writePackManifest } from '../repo/push' +import { storeArtifact, type PlatformQuestion, type StoredArtifact, type UploadEvent } from './upload' +import type { StoragePolicy, StorageProfile } from './profiles' + +export * from './profiles' +export * from './store' +export { corsFix, kuboCorsLines, type CorsFix } from './cors' +export { probeProfile, IPFS_ROWS, S3_ROWS, type ProbeRow, type RowId, type RowState } from './probe' +export { + PlatformDeclinedError, + ReplicationError, + fitManifestUris, + orderUris, + storeArtifact, + type PlatformQuestion, + type StoredArtifact, + type TargetFailure, + type UploadEvent, +} from './upload' + +/** What {@link storeAndRecordPack} did: where the bytes went and the manifest that records them. */ +export interface RecordedPack { + readonly stored: StoredArtifact + readonly manifest: WriteResult +} + +/** + * Upload an artifact under `policy`, then write its `packManifest` — the two steps every + * browser push does per artifact, in that order (the manifest only after the bytes are stored + * and verified). A failure after the upload leaves the bytes stored; a retry re-verifies them + * (content-addressed keys, CIDs, the signer's own chunks) and writes only the manifest. + */ +export async function storeAndRecordPack( + sdk: EvoSDK, + auth: WriteAuth, + repo: V2RepoRef, + bytes: Uint8Array, + meta: { readonly kind: number; readonly objectCount: number; readonly tips?: readonly string[]; readonly supersedes?: readonly string[] }, + opts: { + readonly policy: StoragePolicy | null + readonly profiles: readonly StorageProfile[] + readonly confirmPlatform: (q: PlatformQuestion) => Promise + readonly onStep?: (e: UploadEvent) => void + readonly intent?: string + }, +): Promise { + const stored = await storeArtifact(sdk, auth, repo, bytes, opts) + const manifest = await writePackManifest( + sdk, + auth, + repo, + { + packHash: stored.packHash, + kind: meta.kind, + sizeBytes: stored.sizeBytes, + objectCount: meta.objectCount, + chunkCount: stored.chunkCount, + storage: stored.storage, + uris: stored.uris, + ...(meta.tips ? { tips: meta.tips } : {}), + ...(meta.supersedes ? { supersedes: meta.supersedes } : {}), + }, + opts.intent, + ) + return { stored, manifest } +} diff --git a/forge-web/lib/storage/ipfs.ts b/forge-web/lib/storage/ipfs.ts new file mode 100644 index 000000000..7c1c1c2fb --- /dev/null +++ b/forge-web/lib/storage/ipfs.ts @@ -0,0 +1,168 @@ +/** + * IPFS from the browser: kubo's RPC API (`add` with Forge's pinned import parameters, `pin/ls`, + * `pin/rm`, `id`) and the IPFS Pinning Service API (`POST /pins`, `GET /pins/`). Port of + * forge-core `backends/ipfs.rs`: the CID kubo returns must equal the local re-derivation + * (`./cid`), and the node must report it pinned, before any URI is recorded. + * + * kubo's RPC API refuses cross-origin calls unless `API.HTTPHeaders.Access-Control-Allow-Origin` + * lists this origin (it answers 403 otherwise); the wizard prints the `ipfs config` lines. + */ + +import { cidV1RawLeaves, isCid } from './cid' +import type { ProfilePublic, ProfileSecrets } from './profiles' + +export type IpfsSettings = Extract + +/** The kubo `add` query pinning every import parameter the local CID derivation assumes. */ +export const ADD_PARAMS = + 'cid-version=1&raw-leaves=true&chunker=size-262144&hash=sha2-256&trickle=false&max-file-links=174&pin=true&quieter=true' + +const RPC_TIMEOUT_MS = 60_000 +const PIN_TIMEOUT_MS = 120_000 +const PIN_POLL_MS = 2_000 + +export class IpfsError extends Error { + constructor( + readonly op: string, + message: string, + ) { + super(message) + this.name = 'IpfsError' + } +} + +async function rpc(s: IpfsSettings, secrets: ProfileSecrets, op: string, pathAndQuery: string, body?: FormData): Promise { + const headers = new Headers() + if (secrets.apiAuth) headers.set('authorization', secrets.apiAuth) + const url = `${s.api.replace(/\/+$/, '')}/api/v0/${pathAndQuery}` + let resp: Response + try { + resp = await fetch(url, { method: 'POST', headers, ...(body ? { body } : {}), redirect: 'error', credentials: 'omit', signal: AbortSignal.timeout(RPC_TIMEOUT_MS) }) + } catch (e) { + throw new IpfsError(op, `${op}: the kubo API at ${new URL(url).host} is unreachable or refused this origin (CORS)${e instanceof Error ? ` (${e.message})` : ''}`) + } + const text = await resp.text() + if (!resp.ok) { + const msg = /"Message"\s*:\s*"([^"]{1,200})"/.exec(text)?.[1] ?? text.slice(0, 200) + const hint = resp.status === 403 ? ' — kubo refuses this origin: allow it in API.HTTPHeaders (see the fix below)' : '' + throw new IpfsError(op, `${op} failed: HTTP ${resp.status} ${msg}${hint}`) + } + return text +} + +/** kubo's version (`/api/v0/version`), a cheap reachability check. */ +export async function kuboVersion(s: IpfsSettings, secrets: ProfileSecrets): Promise { + const text = await rpc(s, secrets, 'version', 'version') + return /"Version"\s*:\s*"([^"]+)"/.exec(text)?.[1] ?? 'unknown' +} + +/** Add `bytes` with the pinned parameters; the CID must equal the local derivation and be pinned. */ +export async function addVerified(s: IpfsSettings, secrets: ProfileSecrets, bytes: Uint8Array): Promise { + const form = new FormData() + form.append('file', new Blob([new Uint8Array(bytes)]), 'pack') + const text = await rpc(s, secrets, 'ipfs add', `add?${ADD_PARAMS}`, form) + const line = text.trim().split('\n').pop() ?? '' + const cid = /"Hash"\s*:\s*"([^"]+)"/.exec(line)?.[1] ?? '' + const expected = cidV1RawLeaves(bytes) + if (cid !== expected) { + throw new IpfsError('ipfs add', `kubo returned CID ${cid || '(none)'} but these bytes derive to ${expected} under the pinned import parameters; refusing to record it`) + } + const pinned = await rpc(s, secrets, 'pin check', `pin/ls?arg=${cid}&type=recursive`).catch((e: unknown) => { + if (e instanceof IpfsError && /not pinned/.test(e.message)) return '' + throw e + }) + if (!pinned.includes(cid)) throw new IpfsError('pin check', `kubo added ${cid} but does not report it pinned; a gc would drop it`) + return cid +} + +/** Unpin `cid` (the wizard's probe cleanup). A CID that is not pinned is fine. */ +export async function unpin(s: IpfsSettings, secrets: ProfileSecrets, cid: string): Promise { + if (!isCid(cid)) throw new IpfsError('unpin', 'not a CID') + await rpc(s, secrets, 'unpin', `pin/rm?arg=${cid}`).catch((e: unknown) => { + if (e instanceof IpfsError && /not pinned/.test(e.message)) return '' + throw e + }) +} + +/** The node's announced multiaddrs, handed to a pinning service as `origins`. */ +async function origins(s: IpfsSettings, secrets: ProfileSecrets): Promise { + try { + const text = await rpc(s, secrets, 'id', 'id') + const parsed: unknown = JSON.parse(text) + const addrs = parsed && typeof parsed === 'object' ? (parsed as { Addresses?: unknown }).Addresses : undefined + return Array.isArray(addrs) ? addrs.filter((a): a is string => typeof a === 'string').slice(0, 20) : [] + } catch { + return [] + } +} + +interface PinStatus { + readonly requestid: string + readonly status: string +} + +async function psa(s: IpfsSettings, secrets: ProfileSecrets, op: string, path: string, init: RequestInit = {}): Promise { + if (!secrets.pinningToken) throw new IpfsError(op, 'the pinning service needs an access token') + const url = `${s.pinningEndpoint.replace(/\/+$/, '')}${path}` + let resp: Response + try { + resp = await fetch(url, { + ...init, + headers: { authorization: `Bearer ${secrets.pinningToken}`, 'content-type': 'application/json' }, + redirect: 'error', + credentials: 'omit', + signal: AbortSignal.timeout(RPC_TIMEOUT_MS), + }) + } catch (e) { + throw new IpfsError(op, `${op}: the pinning service is unreachable or refused this origin (CORS)${e instanceof Error ? ` (${e.message})` : ''}`) + } + if (!resp.ok) { + const hint = resp.status === 401 || resp.status === 403 ? ' — check the access token' : '' + throw new IpfsError(op, `${op} failed: HTTP ${resp.status}${hint}`) + } + return resp.json() as Promise +} + +function pinStatus(v: unknown): PinStatus { + const o = v && typeof v === 'object' ? (v as Record) : {} + const requestid = typeof o['requestid'] === 'string' ? o['requestid'] : '' + const status = typeof o['status'] === 'string' ? o['status'] : '' + if (!/^[A-Za-z0-9_-]{1,128}$/.test(requestid)) throw new IpfsError('pin', 'the pinning service returned a malformed request id') + return { requestid, status } +} + +/** An authenticated no-op against the pinning service (`GET /pins?limit=1`). */ +export async function pinningReachable(s: IpfsSettings, secrets: ProfileSecrets): Promise { + await psa(s, secrets, 'pinning service', '/pins?limit=1') +} + +/** + * Pin `cid` on the pinning service and wait for `pinned`. An existing request for the CID is + * reused (a re-push pays nothing twice). + */ +export async function remotePin(s: IpfsSettings, secrets: ProfileSecrets, cid: string, name: string): Promise { + if (!isCid(cid)) throw new IpfsError('pin', 'not a CID') + const found = await psa(s, secrets, 'pin lookup', `/pins?cid=${cid}&status=queued,pinning,pinned`) + const results = found && typeof found === 'object' ? (found as { results?: unknown }).results : undefined + const existing = Array.isArray(results) ? results.map(pinStatus) : [] + if (existing.some((p) => p.status === 'pinned')) return + let pin = existing.find((p) => p.status === 'queued' || p.status === 'pinning') + if (!pin) { + pin = pinStatus(await psa(s, secrets, 'pin', '/pins', { method: 'POST', body: JSON.stringify({ cid, name, origins: await origins(s, secrets) }) })) + } + const deadline = Date.now() + PIN_TIMEOUT_MS + for (;;) { + if (pin.status === 'pinned') return + if (pin.status === 'failed') throw new IpfsError('pin', `the pinning service reports request ${pin.requestid} failed`) + if (Date.now() >= deadline) { + throw new IpfsError('pin', `the pin is still ${pin.status} after ${PIN_TIMEOUT_MS / 1000}s; the service may not be able to reach your kubo node`) + } + await new Promise((r) => setTimeout(r, PIN_POLL_MS)) + pin = pinStatus(await psa(s, secrets, 'pin status', `/pins/${pin.requestid}`)) + } +} + +/** The gateway URL of `cid` on `gateway`. */ +export function gatewayUrl(gateway: string, cid: string): string { + return `${gateway.replace(/\/+$/, '')}/ipfs/${cid}` +} diff --git a/forge-web/lib/storage/probe.ts b/forge-web/lib/storage/probe.ts new file mode 100644 index 000000000..4eeb19de5 --- /dev/null +++ b/forge-web/lib/storage/probe.ts @@ -0,0 +1,233 @@ +/** + * The storage wizard's live test (`ux-dx-spec.md` §3.1 step 3), run from this page, so it + * checks exactly what the browser will do later — including CORS, which a CLI test cannot see. + * + * S3 rows: signed PUT → signed GET → anonymous GET via the public URL → CORS for ranged reads + * (GET + Range, `Content-Range` visible) → CORS for browser pushes (PUT) → delete probe. + * IPFS rows: kubo API → add + CID + pin → gateway re-read (Range) → pinning service → unpin. + * + * A page cannot see WHY a cross-origin request failed: a CORS refusal and a dead host both + * reject `fetch`. {@link reachable} tells them apart with an opaque `no-cors` request, which + * resolves whenever the host answers at all, so a red row says "blocked by CORS" only when the + * host is up. + */ + +import { cidV1RawLeaves } from './cid' +import { S3Error, deleteObject, getObject, publicObjectUrl, putObject, type S3Settings } from './s3' +import { IpfsError, addVerified, gatewayUrl, kuboVersion, pinningReachable, unpin, type IpfsSettings } from './ipfs' +import { normalizedPrefix, type ProfileSecrets, type StorageProfile } from './profiles' +import { sha256Hex } from './sigv4' + +export type RowId = 'put' | 'get' | 'public' | 'cors-range' | 'cors-put' | 'delete' | 'api' | 'add' | 'gateway' | 'pinning' | 'unpin' +export type RowState = 'pending' | 'running' | 'ok' | 'fail' | 'skipped' + +/** One row of the live test. `cors` marks a failure the CORS fix block addresses. */ +export interface ProbeRow { + readonly id: RowId + readonly label: string + readonly state: RowState + readonly detail: string + readonly cors?: boolean +} + +export const S3_ROWS: readonly { id: RowId; label: string }[] = [ + { id: 'put', label: 'signed PUT' }, + { id: 'get', label: 'signed GET' }, + { id: 'public', label: 'anonymous GET via public URL' }, + { id: 'cors-range', label: 'CORS preflight (GET, Range)' }, + { id: 'cors-put', label: 'CORS preflight (PUT) for browser pushes' }, + { id: 'delete', label: 'delete probe' }, +] + +export const IPFS_ROWS: readonly { id: RowId; label: string }[] = [ + { id: 'api', label: 'kubo API' }, + { id: 'add', label: 'add + CID check + pin' }, + { id: 'gateway', label: 'gateway re-read (Range)' }, + { id: 'pinning', label: 'pinning service' }, + { id: 'unpin', label: 'unpin probe' }, +] + +/** Whether `url`'s host answers at all (an opaque no-cors request; never readable). */ +export async function reachable(url: string): Promise { + try { + await fetch(url, { mode: 'no-cors', credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(10_000) }) + return true + } catch { + return false + } +} + +/** A unique probe body, so a stale cached object can never pass for this run's upload. */ +function probeBody(): Uint8Array { + const nonce = Array.from(crypto.getRandomValues(new Uint8Array(12)), (b) => b.toString(16).padStart(2, '0')).join('') + return new TextEncoder().encode(`dash-forge web storage probe ${Date.now()} ${nonce}\n`) +} + +function equal(a: Uint8Array, b: Uint8Array): boolean { + return a.length === b.length && a.every((x, i) => x === b[i]) +} + +type Report = (id: RowId, state: RowState, detail: string, cors?: boolean) => void + +/** Run the S3 test, reporting each row as it settles. Resolves with whether every row passed. */ +export async function probeS3(s: S3Settings, secrets: ProfileSecrets, report: Report): Promise { + const body = probeBody() + const key = `${normalizedPrefix(s.prefix)}probe/forge-web-test-${await sha256Hex(body)}.txt` + let ok = true + const fail = (id: RowId, detail: string, cors = false): void => { + ok = false + report(id, 'fail', detail, cors) + } + const skipRest = (from: RowId[], why: string): void => from.forEach((id) => report(id, 'skipped', why)) + + // 1 + 5: a PUT that lands proves the PUT preflight passed; one refused before any answer is + // CORS when the endpoint is reachable. + report('put', 'running', '') + try { + await putObject(s, secrets, key, body, 'text/plain') + report('put', 'ok', `wrote ${key}`) + report('cors-put', 'ok', 'the browser was allowed to send a signed PUT from this origin') + } catch (e) { + const blocked = e instanceof S3Error && e.status === 0 && (await reachable(s.endpoint)) + if (blocked) { + fail('put', 'the browser refused to send it (see the CORS row)') + fail('cors-put', 'the bucket’s CORS rules do not allow a signed PUT from this origin', true) + } else { + fail('put', errText(e)) + report('cors-put', 'skipped', 'needs a PUT that reaches the bucket') + } + skipRest(['get', 'public', 'cors-range', 'delete'], 'needs the probe object') + return false + } + + report('get', 'running', '') + try { + const got = await getObject(s, secrets, key) + if (equal(got, body)) report('get', 'ok', 'read back byte for byte') + else fail('get', 'the bucket returned different bytes (a cache or proxy in front of it?)') + } catch (e) { + fail('get', errText(e), e instanceof S3Error && e.status === 0) + } + + const publicUrl = publicObjectUrl(s, key) + report('public', 'running', '') + let publicOk = false + try { + const resp = await fetch(publicUrl, { credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(20_000) }) + const got = new Uint8Array(await resp.arrayBuffer()) + if (!resp.ok) fail('public', `HTTP ${resp.status} from ${new URL(publicUrl).host}: the objects are not publicly readable`) + else if (!equal(got, body)) fail('public', `${new URL(publicUrl).host} served different bytes (check the public URL)`) + else { + publicOk = true + report('public', 'ok', `anonymous GET ${new URL(publicUrl).host}`) + } + } catch { + const up = await reachable(publicUrl) + fail('public', up ? 'the browser was not allowed to read it: no Access-Control-Allow-Origin on GET' : `${new URL(publicUrl).host} did not answer`, up) + } + + report('cors-range', 'running', '') + if (!publicOk) report('cors-range', 'skipped', 'needs a public read') + else { + try { + const resp = await fetch(publicUrl, { headers: { Range: 'bytes=0-9' }, credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(20_000) }) + const range = resp.headers.get('content-range') + if (resp.status !== 206) fail('cors-range', `a ranged GET returned ${resp.status} instead of 206: browsing needs HTTP Range support on the public URL`) + else if (range === null) fail('cors-range', 'Content-Range is not exposed to scripts (Access-Control-Expose-Headers)', true) + else report('cors-range', 'ok', 'Range allowed, Content-Range exposed') + } catch { + fail('cors-range', 'the CORS preflight for a Range request was refused: allow the Range request header', true) + } + } + + report('delete', 'running', '') + try { + await deleteObject(s, secrets, key) + report('delete', 'ok', 'probe removed') + } catch (e) { + fail('delete', `${errText(e)}. The probe object (a few bytes) stays at ${key}; delete it by hand.`, e instanceof S3Error && e.status === 0) + } + return ok +} + +/** Run the IPFS test (kubo, plus the pinning service for a pinning profile). */ +export async function probeIpfs(s: IpfsSettings, secrets: ProfileSecrets, report: Report): Promise { + let ok = true + const fail = (id: RowId, detail: string, cors = false): void => { + ok = false + report(id, 'fail', detail, cors) + } + report('api', 'running', '') + try { + report('api', 'ok', `kubo ${await kuboVersion(s, secrets)}`) + } catch (e) { + const up = await reachable(s.api) + fail('api', up ? `${errText(e)}` : `${new URL(s.api).host} did not answer`, up) + for (const id of ['add', 'gateway', 'pinning', 'unpin'] as const) report(id, 'skipped', 'needs the kubo API') + return false + } + + const body = probeBody() + let cid = '' + report('add', 'running', '') + try { + cid = await addVerified(s, secrets, body) + report('add', 'ok', `${cid} matches the local derivation and is pinned`) + } catch (e) { + fail('add', errText(e)) + } + + report('gateway', 'running', '') + if (cid === '' || s.gateway === '') report('gateway', 'skipped', s.gateway === '' ? 'no gateway configured: uploads are verified by CID and pin only' : 'needs the probe') + else { + const url = gatewayUrl(s.gateway, cid) + try { + const resp = await fetch(url, { headers: { Range: `bytes=0-${body.length - 1}` }, credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(20_000) }) + const got = new Uint8Array(await resp.arrayBuffer()) + if (!resp.ok) fail('gateway', `HTTP ${resp.status} from the gateway`) + else if (!equal(got, body)) fail('gateway', 'the gateway served different bytes') + else report('gateway', 'ok', `re-read through ${new URL(url).host}`) + } catch { + const up = await reachable(url) + fail('gateway', up ? 'the gateway refused this origin (CORS)' : 'the gateway did not answer', up) + } + } + + report('pinning', 'running', '') + if (s.kind !== 'ipfs-pinning-service') report('pinning', 'skipped', 'no pinning service in this profile') + else { + try { + await pinningReachable(s, secrets) + report('pinning', 'ok', 'the service accepted the token') + } catch (e) { + fail('pinning', errText(e), e instanceof IpfsError && /CORS/.test(e.message)) + } + } + + report('unpin', 'running', '') + if (cid === '') report('unpin', 'skipped', 'nothing was added') + else { + try { + await unpin(s, secrets, cid) + report('unpin', 'ok', 'probe unpinned (a gc removes it)') + } catch (e) { + fail('unpin', errText(e)) + } + } + return ok +} + +/** Run the test for any profile kind. A Platform profile needs none. */ +export async function probeProfile(p: StorageProfile, report: Report): Promise { + const s = p.settings + if (s.kind === 'platform') return true + if (s.kind === 's3') return probeS3(s, p.secrets, report) + return probeIpfs(s, p.secrets, report) +} + +function errText(e: unknown): string { + return e instanceof Error ? e.message : String(e) +} + +/** The CID a probe body would get (exposed for tests). */ +export const probeCid = cidV1RawLeaves diff --git a/forge-web/lib/storage/profiles.ts b/forge-web/lib/storage/profiles.ts new file mode 100644 index 000000000..20d2659c9 --- /dev/null +++ b/forge-web/lib/storage/profiles.ts @@ -0,0 +1,321 @@ +/** + * Storage profiles in the browser (`ux-dx-spec.md` §3.1): the user's own buckets and IPFS + * nodes, described with the same fields forge-core's `storage.toml` profiles use (camel-cased), + * so a profile means the same thing on both sides. + * + * A profile splits into its public part (endpoint, bucket, public URL: the manifest publishes + * some of it anyway) and its secrets (access key id, secret key, session token, pinning token, + * kubo API auth). Only the public part is ever held in plaintext; the secrets are sealed in + * the vault (`./store`). Nothing here is written on-chain except the public read URLs a push + * records in its `packManifest`. + */ + +import { z } from 'zod' + +/** The kinds a profile can be (parity with forge-core `Profile`). */ +export type ProfileKind = 's3' | 'ipfs-kubo' | 'ipfs-pinning-service' | 'platform' + +/** The provider tiles of the wizard, in the order the spec lists them. */ +export type ProviderId = 'r2' | 'b2' | 'aws' | 'minio' | 'kubo' | 'pinning' | 'platform' + +/** A profile's public settings. */ +export type ProfilePublic = + | { + readonly kind: 's3' + readonly provider: 'r2' | 'b2' | 'aws' | 'minio' + /** API endpoint origin (`https://.r2.cloudflarestorage.com`). */ + readonly endpoint: string + readonly region: string + readonly bucket: string + /** Path-style addressing (`endpoint/bucket/key`); R2, B2 and MinIO want it. */ + readonly pathStyle: boolean + /** Public read origin; objects are read at `/`. */ + readonly publicUrl: string + /** Key prefix inside the bucket (`forge/`), or ''. */ + readonly prefix: string + } + | { + readonly kind: 'ipfs-kubo' | 'ipfs-pinning-service' + readonly provider: 'kubo' | 'pinning' + /** kubo RPC API origin (`http://127.0.0.1:5001`). */ + readonly api: string + /** A gateway serving this node's content, for the verification re-read. */ + readonly gateway: string + /** A public https gateway to record next to `ipfs://` (optional). */ + readonly publicGateway: string + /** Pinning Service API base URL (pinning profiles only). */ + readonly pinningEndpoint: string + } + | { readonly kind: 'platform'; readonly provider: 'platform' } + +/** The secrets a profile needs, sealed in the vault. */ +export interface ProfileSecrets { + readonly accessKeyId?: string + readonly secretAccessKey?: string + readonly sessionToken?: string + /** The full `Authorization` header value for a kubo RPC API behind auth. */ + readonly apiAuth?: string + /** A Pinning Service API access token. */ + readonly pinningToken?: string +} + +/** A named profile: public settings plus (when unlocked) its secrets. */ +export interface StorageProfile { + readonly name: string + readonly settings: ProfilePublic + readonly secrets: ProfileSecrets +} + +/** One provider tile: what the form asks for and where to find it. */ +export interface ProviderPreset { + readonly id: ProviderId + readonly title: string + readonly blurb: string + readonly kind: ProfileKind + /** Defaults the form starts from. */ + readonly defaults: Partial> + /** Field → "where to find this" hint (with a console link where one exists). */ + readonly hints: Readonly> +} + +export const PROVIDERS: readonly ProviderPreset[] = [ + { + id: 'r2', + title: 'Cloudflare R2', + blurb: 'recommended: free egress', + kind: 's3', + defaults: { region: 'auto', pathStyle: true, endpoint: 'https://.r2.cloudflarestorage.com' }, + hints: { + endpoint: { text: 'R2 → Overview → Account details → S3 API. Use the origin only, without the bucket.', href: 'https://dash.cloudflare.com/?to=/:account/r2/overview' }, + bucket: { text: 'R2 → Create bucket, e.g. forge.' }, + publicUrl: { text: 'Bucket → Settings → Public access: the r2.dev subdomain or your custom domain.' }, + accessKeyId: { text: 'R2 → Manage R2 API Tokens → Create API token (Object Read & Write, this bucket).', href: 'https://dash.cloudflare.com/?to=/:account/r2/api-tokens' }, + region: { text: 'R2 always uses auto.' }, + }, + }, + { + id: 'b2', + title: 'Backblaze B2', + blurb: 'cheap storage, S3 API', + kind: 's3', + defaults: { region: 'us-west-004', pathStyle: true, endpoint: 'https://s3.us-west-004.backblazeb2.com' }, + hints: { + endpoint: { text: 'Buckets → your bucket → Endpoint. The region is its middle part.', href: 'https://secure.backblaze.com/b2_buckets.htm' }, + bucket: { text: 'Create the bucket with Files in bucket: Public.' }, + publicUrl: { text: 'https://s3..backblazeb2.com/ works for public buckets.' }, + accessKeyId: { text: 'App Keys → Add a New Application Key (Read and Write, this bucket). keyID is the access key id.', href: 'https://secure.backblaze.com/app_keys.htm' }, + }, + }, + { + id: 'aws', + title: 'AWS S3', + blurb: 'the original', + kind: 's3', + defaults: { region: 'us-east-1', pathStyle: false, endpoint: 'https://s3.us-east-1.amazonaws.com' }, + hints: { + endpoint: { text: 'https://s3..amazonaws.com for the bucket’s region.', href: 'https://s3.console.aws.amazon.com/s3/buckets' }, + bucket: { text: 'A bucket name with dots needs path-style addressing over TLS.' }, + publicUrl: { text: 'https://.s3..amazonaws.com with a public-read bucket policy, or a CloudFront URL.' }, + accessKeyId: { text: 'IAM → a user or role limited to s3:PutObject, GetObject, DeleteObject on this bucket.', href: 'https://console.aws.amazon.com/iam/home#/users' }, + }, + }, + { + id: 'minio', + title: 'MinIO / other S3', + blurb: 'self-hosted or any S3-compatible store', + kind: 's3', + defaults: { region: 'us-east-1', pathStyle: true, endpoint: 'https://minio.example.org' }, + hints: { + endpoint: { text: 'Your server’s S3 API origin (local MinIO: http://127.0.0.1:9000).' }, + bucket: { text: 'mc mb /forge, then mc anonymous set download /forge.' }, + publicUrl: { text: 'Usually /.' }, + accessKeyId: { text: 'A MinIO user or access key with read and write on the bucket.' }, + }, + }, + { + id: 'kubo', + title: 'IPFS (kubo)', + blurb: 'your own IPFS node', + kind: 'ipfs-kubo', + defaults: { api: 'http://127.0.0.1:5001', gateway: 'http://127.0.0.1:8080' }, + hints: { + api: { text: 'The kubo RPC API (Addresses.API). The browser calls it directly, so it needs CORS for this origin.' }, + gateway: { text: 'A gateway serving this node’s content (Addresses.Gateway), used to re-read uploads.' }, + publicGateway: { text: 'Optional: a public https gateway for your node, recorded so browsers can fetch directly.' }, + }, + }, + { + id: 'pinning', + title: 'IPFS pinning service', + blurb: 'kubo + a remote pin', + kind: 'ipfs-pinning-service', + defaults: { api: 'http://127.0.0.1:5001', gateway: 'http://127.0.0.1:8080' }, + hints: { + api: { text: 'The kubo node the content is added through; the service fetches it from there.' }, + pinningEndpoint: { text: 'The service’s Pinning Service API base URL (for example Pinata’s https://api.pinata.cloud/psa).', href: 'https://ipfs.github.io/pinning-services-api-spec/' }, + pinningToken: { text: 'An access token from the service’s dashboard.' }, + }, + }, + { + id: 'platform', + title: 'Dash Platform', + blurb: 'permanent, ~0.28 DASH per MiB', + kind: 'platform', + defaults: {}, + hints: {}, + }, +] as const + +/** The preset for `id`. */ +export function providerPreset(id: ProviderId): ProviderPreset { + const p = PROVIDERS.find((x) => x.id === id) + if (!p) throw new Error(`unknown provider ${id}`) + return p +} + +/** A valid profile name: it becomes part of a policy list, so no commas or whitespace. */ +export function validProfileName(name: string): boolean { + return /^[A-Za-z0-9._-]{1,64}$/.test(name) +} + +/** An http(s) origin or URL with no userinfo, query or fragment (they could smuggle a token). */ +function checkUrl(field: string, value: string, opts: { originOnly?: boolean } = {}): string | null { + let url: URL + try { + url = new URL(value) + } catch { + return `${field} is not a URL` + } + if (url.protocol !== 'https:' && url.protocol !== 'http:') return `${field} must be an http(s) URL` + if (url.username || url.password) return `${field} must not carry a user name or password` + if (url.search || url.hash) return `${field} must not carry a query or fragment` + if (opts.originOnly && url.pathname !== '/' && url.pathname !== '') return `${field} must be an origin (scheme://host[:port]) with no path` + if (url.protocol === 'http:' && !isLoopback(url.hostname)) return `${field} must be https (plain http is allowed only for a node on this machine)` + return null +} + +/** Whether a hostname is this machine (the only place a browser may reach over plain http). */ +export function isLoopback(hostname: string): boolean { + return hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '[::1]' +} + +/** Why a profile is not usable, or null. Parity with forge-core `S3Config::validate`. */ +export function profileProblem(p: StorageProfile): string | null { + if (!validProfileName(p.name)) return "use a name of letters, digits, '.', '_' and '-' (max 64)" + const s = p.settings + if (s.kind === 'platform') return null + if (s.kind === 's3') { + const e = checkUrl('the endpoint', s.endpoint, { originOnly: true }) + if (e) return e + if (!/^[a-z0-9._-]+$/.test(s.bucket)) return "the bucket name must be lowercase letters, digits, '-', '.', '_'" + const host = new URL(s.endpoint).hostname + if (!s.pathStyle && (/^\d+\.\d+\.\d+\.\d+$/.test(host) || host.startsWith('['))) return 'an IP-address endpoint needs path-style addressing' + if (!s.pathStyle && s.bucket.includes('.') && s.endpoint.startsWith('https:')) return "a bucket name with '.' breaks TLS for virtual-hosted addressing; use path-style" + const prefix = s.prefix.replace(/\/+$/, '') + if (prefix !== '' && prefix.split('/').some((seg) => seg === '' || seg === '.' || seg === '..')) return "the prefix has an empty, '.' or '..' segment" + if (s.publicUrl === '') return 'a public URL is needed: browsers read packs anonymously from it' + const pu = checkUrl('the public URL', s.publicUrl) + if (pu) return pu + if (!p.secrets.accessKeyId || !p.secrets.secretAccessKey) return 'an access key id and secret are needed to sign uploads' + return null + } + const api = checkUrl('the kubo API', s.api, { originOnly: true }) + if (api) return api + if (s.gateway !== '') { + const gw = checkUrl('the gateway', s.gateway, { originOnly: true }) + if (gw) return gw + } + if (s.publicGateway !== '') { + const pg = checkUrl('the public gateway', s.publicGateway, { originOnly: true }) + if (pg) return pg + } + if (s.kind === 'ipfs-pinning-service') { + const pe = checkUrl('the pinning endpoint', s.pinningEndpoint) + if (pe) return pe + if (!p.secrets.pinningToken) return 'the pinning service needs an access token' + } + return null +} + +/** The key prefix normalized: '' or ending in exactly one `/` (parity with forge-core). */ +export function normalizedPrefix(prefix: string): string { + const p = prefix.replace(/^\/+|\/+$/g, '') + return p === '' ? '' : `${p}/` +} + +/** The content-addressed object key of an artifact (`packs/.pack`, as the CLI writes). */ +export function artifactKey(settings: Extract, sha256Hex: string): string { + return `${normalizedPrefix(settings.prefix)}packs/${sha256Hex}.pack` +} + +// --------------------------------------------------------------------------- +// Persisted shapes (zod: what comes back from IndexedDB is parsed, not cast) +// --------------------------------------------------------------------------- + +const httpString = z.string().max(300) + +export const profilePublicSchema: z.ZodType = z.discriminatedUnion('kind', [ + z.object({ + kind: z.literal('s3'), + provider: z.enum(['r2', 'b2', 'aws', 'minio']), + endpoint: httpString, + region: z.string().max(64), + bucket: z.string().max(63), + pathStyle: z.boolean(), + publicUrl: httpString, + prefix: z.string().max(200), + }), + z.object({ + kind: z.enum(['ipfs-kubo', 'ipfs-pinning-service']), + provider: z.enum(['kubo', 'pinning']), + api: httpString, + gateway: httpString, + publicGateway: httpString, + pinningEndpoint: httpString, + }), + z.object({ kind: z.literal('platform'), provider: z.literal('platform') }), +]) as z.ZodType + +export const profileSecretsSchema: z.ZodType = z + .object({ + accessKeyId: z.string().max(256).optional(), + secretAccessKey: z.string().max(512).optional(), + sessionToken: z.string().max(4096).optional(), + apiAuth: z.string().max(1024).optional(), + pinningToken: z.string().max(4096).optional(), + }) + .strict() + +/** A repo's (or the default) browser-push storage policy. */ +export interface StoragePolicy { + /** Profile names, in order. Empty: no storage configured. */ + readonly targets: readonly string[] + /** Copies that must confirm (1..targets). */ + readonly replicas: number + /** Store on Platform when the targets cannot confirm `replicas` (asks, with the price, first). */ + readonly platformFallback: boolean +} + +export const policySchema: z.ZodType = z.object({ + targets: z.array(z.string().max(64)).max(8), + replicas: z.number().int().min(1).max(8), + platformFallback: z.boolean(), +}) + +/** The replication choices of the wizard (§3.1 step 4). */ +export type ReplicationChoice = 'one' | 'all' | 'fallback' + +/** A policy for `targets` under a replication choice. */ +export function policyFor(targets: readonly string[], choice: ReplicationChoice): StoragePolicy { + return { + targets: [...targets], + replicas: choice === 'all' ? Math.max(1, targets.length) : 1, + platformFallback: choice === 'fallback', + } +} + +/** Which replication choice a policy corresponds to (for the radio). */ +export function choiceOf(policy: StoragePolicy): ReplicationChoice { + if (policy.platformFallback) return 'fallback' + return policy.replicas > 1 && policy.replicas >= policy.targets.length ? 'all' : 'one' +} diff --git a/forge-web/lib/storage/s3.ts b/forge-web/lib/storage/s3.ts new file mode 100644 index 000000000..9fc26a178 --- /dev/null +++ b/forge-web/lib/storage/s3.ts @@ -0,0 +1,155 @@ +/** + * A minimal S3 client for the browser: signed PUT / GET / HEAD / DELETE of one object, and + * anonymous reads through the bucket's public URL. Port of the request shapes of forge-core + * `backends/s3.rs` (path-style or virtual-hosted URLs, the key percent-encoded once, the body + * hash in `x-amz-content-sha256` so the store verifies the upload server-side). + * + * Browser specifics: `fetch` sets `Host` from the URL, so the signed host must be exactly the + * URL's; `redirect: 'error'` so a signed request is never replayed to another host; and every + * header the request carries beyond the CORS-safelisted ones (authorization, x-amz-*) needs the + * bucket's CORS rules to allow it, which the wizard tests before a push depends on it. + */ + +import { EMPTY_PAYLOAD_SHA256, amzDate, keyHasBadSegment, sha256Hex, signRequest, uriEncode } from './sigv4' +import type { ProfilePublic, ProfileSecrets } from './profiles' + +export type S3Settings = Extract + +/** How long one S3 request may take before it is abandoned. */ +export const S3_TIMEOUT_MS = 60_000 + +/** An S3 request that failed: the operation, the HTTP status (0: no response) and why. */ +export class S3Error extends Error { + constructor( + readonly op: string, + readonly status: number, + message: string, + ) { + super(message) + this.name = 'S3Error' + } +} + +/** The API URL of `key` (path-style: `endpoint/bucket/key`; else `bucket.host/key`). */ +export function objectUrl(s: S3Settings, key: string): URL { + if (keyHasBadSegment(key)) throw new S3Error('url', 0, `object key ${JSON.stringify(key)} has an empty, '.' or '..' segment`) + const endpoint = new URL(s.endpoint) + const encoded = uriEncode(key, true) + const url = s.pathStyle + ? new URL(`${endpoint.origin}/${s.bucket}/${encoded}`) + : new URL(`${endpoint.protocol}//${s.bucket}.${endpoint.host}/${encoded}`) + // What is signed must be what is sent: the parser must keep our encoding verbatim. + const intended = s.pathStyle ? `/${s.bucket}/${encoded}` : `/${encoded}` + if (url.pathname !== intended) throw new S3Error('url', 0, 'the request path would differ from the signed one; refusing to sign it') + return url +} + +/** The anonymous public URL of `key`. */ +export function publicObjectUrl(s: S3Settings, key: string): string { + return `${s.publicUrl.replace(/\/+$/, '')}/${uriEncode(key, true)}` +} + +/** The `s3://bucket/key` locator a manifest records next to the public URL (CLI read path). */ +export function s3Uri(s: S3Settings, key: string): string { + return `s3://${s.bucket}/${key}` +} + +async function send( + s: S3Settings, + secrets: ProfileSecrets, + op: string, + method: 'GET' | 'PUT' | 'HEAD' | 'DELETE', + key: string, + opts: { body?: Uint8Array; contentType?: string; range?: string; signal?: AbortSignal } = {}, +): Promise { + const url = objectUrl(s, key) + const payloadHash = opts.body ? await sha256Hex(opts.body) : EMPTY_PAYLOAD_SHA256 + const extra: [string, string][] = [] + if (opts.contentType) extra.push(['content-type', opts.contentType]) + if (opts.range) extra.push(['range', opts.range]) + const headers = new Headers(extra) + if (!secrets.accessKeyId || !secrets.secretAccessKey) throw new S3Error(op, 0, 'no credentials: add the access key id and secret') + const signed = await signRequest( + { + method, + host: url.host, + canonicalUri: url.pathname, + headers: extra, + payloadHash, + region: s.region, + service: 's3', + contentSha256Header: true, + }, + { + accessKeyId: secrets.accessKeyId, + secretAccessKey: secrets.secretAccessKey, + ...(secrets.sessionToken ? { sessionToken: secrets.sessionToken } : {}), + }, + amzDate(), + ) + for (const [k, v] of signed) headers.set(k, v) + const timeout = AbortSignal.timeout(S3_TIMEOUT_MS) + const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout + try { + return await fetch(url, { + method, + headers, + ...(opts.body ? { body: new Uint8Array(opts.body) } : {}), + redirect: 'error', + credentials: 'omit', + cache: 'no-store', + signal, + }) + } catch (e) { + // A CORS refusal and a network failure look the same to a page: say which is likely. + const why = timeout.aborted ? `no answer within ${S3_TIMEOUT_MS / 1000}s` : 'the request was blocked or the host is unreachable (a CORS rule missing for this origin, or the endpoint is down)' + throw new S3Error(op, 0, `${op} ${url.host}: ${why}${e instanceof Error && !timeout.aborted ? ` (${e.message})` : ''}`) + } +} + +async function failed(op: string, resp: Response): Promise { + let code = '' + try { + const text = (await resp.text()).slice(0, 2000) + code = /([^<]{1,64})<\/Code>/.exec(text)?.[1] ?? '' + } catch { + /* body unreadable */ + } + const hint = + resp.status === 403 + ? ' — check the key, its bucket permissions, the region (R2 wants auto) and the clock' + : resp.status === 404 + ? ' — check the bucket name and endpoint' + : resp.status === 301 || resp.status === 307 + ? ' — the bucket lives in another region; set that endpoint' + : '' + return new S3Error(op, resp.status, `${op} failed: HTTP ${resp.status}${code ? ` ${code}` : ''}${hint}`) +} + +/** Signed PUT of `bytes` at `key`. */ +export async function putObject(s: S3Settings, secrets: ProfileSecrets, key: string, bytes: Uint8Array, contentType = 'application/octet-stream', signal?: AbortSignal): Promise { + const resp = await send(s, secrets, 'signed PUT', 'PUT', key, { body: bytes, contentType, ...(signal ? { signal } : {}) }) + if (!resp.ok) throw await failed('signed PUT', resp) +} + +/** Signed GET of `key` (whole object, or a `Range`). */ +export async function getObject(s: S3Settings, secrets: ProfileSecrets, key: string, range?: string, signal?: AbortSignal): Promise { + const resp = await send(s, secrets, 'signed GET', 'GET', key, { ...(range ? { range } : {}), ...(signal ? { signal } : {}) }) + if (!resp.ok) throw await failed('signed GET', resp) + return new Uint8Array(await resp.arrayBuffer()) +} + +/** Signed HEAD: the object's size, or null when it does not exist. */ +export async function headObject(s: S3Settings, secrets: ProfileSecrets, key: string): Promise { + const resp = await send(s, secrets, 'signed HEAD', 'HEAD', key) + if (resp.status === 404) return null + if (!resp.ok) throw await failed('signed HEAD', resp) + const len = Number(resp.headers.get('content-length')) + return Number.isFinite(len) ? len : 0 +} + +/** Signed DELETE of `key` (idempotent: a missing object is fine). */ +export async function deleteObject(s: S3Settings, secrets: ProfileSecrets, key: string): Promise { + const resp = await send(s, secrets, 'delete', 'DELETE', key) + if (!resp.ok && resp.status !== 404) throw await failed('delete', resp) +} diff --git a/forge-web/lib/storage/sigv4.test.ts b/forge-web/lib/storage/sigv4.test.ts new file mode 100644 index 000000000..17824916f --- /dev/null +++ b/forge-web/lib/storage/sigv4.test.ts @@ -0,0 +1,160 @@ +/** + * SigV4 parity with forge-core `backends/sigv4.rs`: the same AWS test-suite vectors and S3 + * API reference examples, so the browser and the CLI sign byte for byte alike. + */ + +import { describe, expect, it } from 'vitest' + +import { + EMPTY_PAYLOAD_SHA256, + amzDate, + canonicalHeaders, + canonicalQuery, + keyHasBadSegment, + sha256Hex, + signRequest, + uriEncode, + type SigningKeys, +} from './sigv4' + +const SUITE: SigningKeys = { accessKeyId: 'AKIDEXAMPLE', secretAccessKey: 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY' } +const SUITE_DATE = { date: '20150830', datetime: '20150830T123600Z' } + +async function suiteSign( + method: string, + path: string, + query: [string, string][] = [], + headers: [string, string][] = [], + sessionToken?: string, +): Promise { + const out = await signRequest( + { + method, + host: 'example.amazonaws.com', + canonicalUri: uriEncode(path, true), + query, + headers, + payloadHash: EMPTY_PAYLOAD_SHA256, + region: 'us-east-1', + service: 'service', + contentSha256Header: false, + }, + sessionToken ? { ...SUITE, sessionToken } : SUITE, + SUITE_DATE, + ) + const auth = out.find(([k]) => k === 'authorization')?.[1] ?? '' + return auth.split('Signature=')[1] ?? '' +} + +describe('SigV4 test suite (generic service)', () => { + it.each([ + ['get-vanilla', 'GET', '/', [], [], '5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31'], + ['get-space-unnormalized', 'GET', '/example space/', [], [], '652487583200325589f1fba4c7e578f72c47cb61beeca81406b39ddec1366741'], + ['get-utf8', 'GET', '/ሴ', [], [], '8318018e0b0f223aa2bbf98705b62bb787dc9c0e678f255a891fd03141be5d85'], + [ + 'get-unreserved', + 'GET', + '/-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz', + [], + [], + '07ef7494c76fa4850883e2b006601f940f8a34d404d0cfa977f52a65bbf5f24f', + ], + ['query-order-key-case', 'GET', '/', [['Param2', 'value2'], ['Param1', 'value1']], [], 'b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500'], + ['utf8-query-key', 'GET', '/', [['ሴ', 'bar']], [], '2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04'], + ['post-header-key-sort', 'POST', '/', [], [['My-Header1', 'value1']], 'c5410059b04c1ee005303aed430f6e6645f61f4dc9e1461ec8f8916fdf18852c'], + ] as const)('%s', async (_name, method, path, query, headers, want) => { + expect(await suiteSign(method, path, [...query] as [string, string][], [...headers] as [string, string][])).toBe(want) + }) + + it('signs the session token', async () => { + expect(await suiteSign('GET', '/', [], [], '6e86291e8372ff2a2260956d9b8aae1d763fbf315fa00fa31553b73ebf194267')).toBe( + '07ec1639c89043aa0e3e2de82b96708f198cceab042d4a97044c66dd9f74e7f8', + ) + }) +}) + +describe('S3 API reference examples', () => { + const S3: SigningKeys = { accessKeyId: 'AKIAIOSFODNN7EXAMPLE', secretAccessKey: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' } + const DATE = amzDate(Date.UTC(2013, 4, 24)) + + it('formats the UTC stamp', () => { + expect(DATE).toEqual({ date: '20130524', datetime: '20130524T000000Z' }) + expect(amzDate(1_440_938_160_000).datetime).toBe('20150830T123600Z') + expect(amzDate(951_782_400_000).date).toBe('20000229') + }) + + it('GET object with a range', async () => { + const out = await signRequest( + { + method: 'GET', + host: 'examplebucket.s3.amazonaws.com', + canonicalUri: '/test.txt', + headers: [['range', 'bytes=0-9']], + payloadHash: EMPTY_PAYLOAD_SHA256, + region: 'us-east-1', + service: 's3', + contentSha256Header: true, + }, + S3, + DATE, + ) + expect(out.find(([k]) => k === 'authorization')?.[1]).toBe( + 'AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, ' + + 'SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, ' + + 'Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41', + ) + }) + + it('PUT object whose key needs encoding', async () => { + const payload = await sha256Hex(new TextEncoder().encode('Welcome to Amazon S3.')) + expect(payload).toBe('44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072') + const uri = `/${uriEncode('test$file.text', true)}` + expect(uri).toBe('/test%24file.text') + const out = await signRequest( + { + method: 'PUT', + host: 'examplebucket.s3.amazonaws.com', + canonicalUri: uri, + headers: [ + ['Date', 'Fri, 24 May 2013 00:00:00 GMT'], + ['x-amz-storage-class', 'REDUCED_REDUNDANCY'], + ], + payloadHash: payload, + region: 'us-east-1', + service: 's3', + contentSha256Header: true, + }, + S3, + DATE, + ) + expect(out.find(([k]) => k === 'authorization')?.[1]).toMatch( + /SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd$/, + ) + }) +}) + +describe('canonicalization helpers', () => { + it('encodes like forge-core', () => { + expect(uriEncode('a b+c=d&e', true)).toBe('a%20b%2Bc%3Dd%26e') + expect(uriEncode('dir/file', true)).toBe('dir/file') + expect(uriEncode('dir/file', false)).toBe('dir%2Ffile') + expect(uriEncode('ü', true)).toBe('%C3%BC') + expect(uriEncode('100%', true)).toBe('100%25') + }) + + it('trims, collapses and merges headers', () => { + expect(canonicalHeaders([['X-B', ' two words '], ['x-a', '1'], ['X-A', '2']])).toEqual({ + canonical: 'x-a:1,2\nx-b:two words\n', + signed: 'x-a;x-b', + }) + }) + + it('keeps = on an empty query value', () => { + expect(canonicalQuery([['b', ''], ['a', 'x y']])).toBe('a=x%20y&b=') + }) + + it('refuses keys an HTTP stack would normalize', () => { + expect(keyHasBadSegment('packs/abc.pack')).toBe(false) + for (const k of ['', 'a//b', './a', 'a/../b', 'a/']) expect(keyHasBadSegment(k)).toBe(true) + }) +}) diff --git a/forge-web/lib/storage/sigv4.ts b/forge-web/lib/storage/sigv4.ts new file mode 100644 index 000000000..43f0d462f --- /dev/null +++ b/forge-web/lib/storage/sigv4.ts @@ -0,0 +1,164 @@ +/** + * AWS Signature Version 4, for S3-compatible buckets (R2, B2, AWS S3, MinIO), on WebCrypto. + * + * A port of forge-core `backends/sigv4.rs` and pinned by the same vectors: the AWS SigV4 test + * suite and the worked examples in the S3 API reference (`sigv4.test.ts`). S3 canonicalization + * differs from generic SigV4 in one way that matters: the object path is percent-encoded ONCE, + * keeping `/`, and never normalized. A key with an empty, `.` or `..` segment is refused by the + * caller, because `fetch` would normalize it after signing and address another object. + * + * The secret access key is imported into a non-extractable HMAC `CryptoKey` for the first step + * of the key derivation and never formatted, logged or returned. + */ + +/** SHA-256 of the empty string: the payload hash of a body-less request. */ +export const EMPTY_PAYLOAD_SHA256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' + +const ALGORITHM = 'AWS4-HMAC-SHA256' +const enc = new TextEncoder() + +/** The credentials a request is signed with. */ +export interface SigningKeys { + readonly accessKeyId: string + readonly secretAccessKey: string + readonly sessionToken?: string +} + +/** A signing timestamp in the two forms SigV4 uses. */ +export interface AmzDate { + /** `YYYYMMDD` (the credential-scope date). */ + readonly date: string + /** `YYYYMMDDTHHMMSSZ` (the `x-amz-date` value). */ + readonly datetime: string +} + +/** The UTC stamp of `ms` since the epoch. */ +export function amzDate(ms: number = Date.now()): AmzDate { + const iso = new Date(ms).toISOString() // 2013-05-24T00:00:00.000Z + const datetime = `${iso.slice(0, 4)}${iso.slice(5, 7)}${iso.slice(8, 10)}T${iso.slice(11, 13)}${iso.slice(14, 16)}${iso.slice(17, 19)}Z` + return { date: datetime.slice(0, 8), datetime } +} + +/** + * RFC 3986 percent-encoding as SigV4 specifies it: every byte except `A-Z a-z 0-9 - . _ ~` + * becomes `%XX` (uppercase hex). With `keepSlash`, `/` stays (object-key paths). + */ +export function uriEncode(input: string, keepSlash: boolean): string { + let out = '' + for (const b of enc.encode(input)) { + const c = String.fromCharCode(b) + if (/[A-Za-z0-9\-._~]/.test(c) || (keepSlash && c === '/')) out += c + else out += `%${b.toString(16).toUpperCase().padStart(2, '0')}` + } + return out +} + +/** Whether an object key has an empty, `.` or `..` segment (see the module doc). */ +export function keyHasBadSegment(key: string): boolean { + return key === '' || key.split('/').some((s) => s === '' || s === '.' || s === '..') +} + +function hex(bytes: ArrayBuffer | Uint8Array): string { + const u = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes) + let s = '' + for (const b of u) s += b.toString(16).padStart(2, '0') + return s +} + +function buf(b: Uint8Array): Uint8Array { + return new Uint8Array(b) +} + +/** Lowercase-hex SHA-256 of `bytes`. */ +export async function sha256Hex(bytes: Uint8Array): Promise { + return hex(await crypto.subtle.digest('SHA-256', buf(bytes))) +} + +async function hmac(key: Uint8Array | CryptoKey, data: string): Promise { + const k = + key instanceof Uint8Array + ? await crypto.subtle.importKey('raw', buf(key), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']) + : key + return new Uint8Array(await crypto.subtle.sign('HMAC', k, enc.encode(data))) +} + +/** The derived signing key: `HMAC(HMAC(HMAC(HMAC("AWS4"+secret, date), region), service), "aws4_request")`. */ +async function signingKey(secret: string, date: string, region: string, service: string): Promise { + const root = await crypto.subtle.importKey('raw', enc.encode(`AWS4${secret}`), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']) + const kDate = await hmac(root, date) + const kRegion = await hmac(kDate, region) + const kService = await hmac(kRegion, service) + return hmac(kService, 'aws4_request') +} + +/** + * The canonical query: each key and value encoded (slashes too), sorted by encoded key then + * value, joined with `&`. An empty value keeps its `=`. + */ +export function canonicalQuery(pairs: readonly (readonly [string, string])[]): string { + return pairs + .map(([k, v]) => [uriEncode(k, false), uriEncode(v, false)] as const) + .sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0)) + .map(([k, v]) => `${k}=${v}`) + .join('&') +} + +/** + * Canonical headers and the `SignedHeaders` list: names lowercased, values trimmed with inner + * whitespace runs collapsed, sorted by name (stable), duplicates joined by `,`. + */ +export function canonicalHeaders(headers: readonly (readonly [string, string])[]): { canonical: string; signed: string } { + const norm = headers + .map(([k, v]) => [k.trim().toLowerCase(), v.split(/\s+/).filter((p) => p !== '').join(' ')] as const) + .map((h, i) => ({ h, i })) + .sort((a, b) => (a.h[0] < b.h[0] ? -1 : a.h[0] > b.h[0] ? 1 : a.i - b.i)) + .map((x) => x.h) + const merged: [string, string][] = [] + for (const [k, v] of norm) { + const last = merged[merged.length - 1] + if (last && last[0] === k) last[1] = `${last[1]},${v}` + else merged.push([k, v]) + } + return { + canonical: merged.map(([k, v]) => `${k}:${v}\n`).join(''), + signed: merged.map(([k]) => k).join(';'), + } +} + +/** One request to sign. */ +export interface RequestToSign { + readonly method: string + /** The `Host` exactly as sent: `host`, or `host:port` for a non-default port. */ + readonly host: string + /** The already-encoded path ({@link uriEncode} with `keepSlash`). */ + readonly canonicalUri: string + readonly query?: readonly (readonly [string, string])[] + /** Extra headers to sign (`range`, `content-type`); the signer adds its own. */ + readonly headers?: readonly (readonly [string, string])[] + /** Hex SHA-256 of the body, or {@link EMPTY_PAYLOAD_SHA256}. */ + readonly payloadHash: string + readonly region: string + readonly service: string + /** Send and sign `x-amz-content-sha256` (S3 requires it; the generic suite does not). */ + readonly contentSha256Header: boolean +} + +/** + * Sign `req`: the headers to add to the outgoing request, `authorization` last. `host` is + * signed but not returned; `fetch` derives it from the URL, which must produce `req.host`. + */ +export async function signRequest(req: RequestToSign, keys: SigningKeys, when: AmzDate): Promise<[string, string][]> { + const added: [string, string][] = [['x-amz-date', when.datetime]] + if (req.contentSha256Header) added.push(['x-amz-content-sha256', req.payloadHash]) + if (keys.sessionToken) added.push(['x-amz-security-token', keys.sessionToken]) + const all: [string, string][] = [['host', req.host], ...(req.headers ?? []).map(([k, v]) => [k, v] as [string, string]), ...added] + const { canonical, signed } = canonicalHeaders(all) + const creq = [req.method, req.canonicalUri, canonicalQuery(req.query ?? []), canonical, signed, req.payloadHash].join('\n') + const scope = `${when.date}/${req.region}/${req.service}/aws4_request` + const sts = [ALGORITHM, when.datetime, scope, await sha256Hex(enc.encode(creq))].join('\n') + const key = await signingKey(keys.secretAccessKey, when.date, req.region, req.service) + const signature = hex(await hmac(key, sts)) + key.fill(0) + added.push(['authorization', `${ALGORITHM} Credential=${keys.accessKeyId}/${scope}, SignedHeaders=${signed}, Signature=${signature}`]) + return added +} diff --git a/forge-web/lib/storage/storage.test.ts b/forge-web/lib/storage/storage.test.ts new file mode 100644 index 000000000..91fb1a22a --- /dev/null +++ b/forge-web/lib/storage/storage.test.ts @@ -0,0 +1,289 @@ +/** + * The storage layer, offline: profile validation, policies, URI ordering and budgets, chunk + * splitting (parity with forge-core `pack::split`), the CORS fix blocks, the config sealed in + * the vault, and the replication engine against in-memory S3 and kubo fakes. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { resetMemoryStores, idbEntries } from '../idb' +import { lockVault, storeInVault, unlockWithPassphrase } from '../auth/vault' +import { splitChunks, manifestUrisProblem } from '../repo/push' +import { CHUNK_PAYLOAD_MAX, FIELD_MAX } from '../constants' +import { corsFix } from './cors' +import { cidV1RawLeaves } from './cid' +import { + artifactKey, + choiceOf, + policyFor, + profileProblem, + type ProfilePublic, + type StorageProfile, +} from './profiles' +import { + EMPTY_STORAGE_CONFIG, + loadStorageConfig, + policyForRepo, + policyProblem, + saveStorageConfig, + withProfile, + withRepoPolicy, + withoutProfile, +} from './store' +import { ReplicationError, PlatformDeclinedError, fitManifestUris, orderUris, storeArtifact } from './upload' +import { sha256Hex } from './sigv4' +import type { WriteAuth } from '../sdk' +import type { V2RepoRef } from '../repo/contract' +import type { EvoSDK } from '@dashevo/evo-sdk' + +const ID = '9r27eDsuXEqoMNymW1A2MKFrpBhzSkepVKwXrGzq9dUD' + +const S3: StorageProfile = { + name: 'minio', + settings: { + kind: 's3', + provider: 'minio', + endpoint: 'http://127.0.0.1:9000', + region: 'us-east-1', + bucket: 'forge-byo', + pathStyle: true, + publicUrl: 'http://127.0.0.1:9000/forge-byo', + prefix: 'web', + }, + secrets: { accessKeyId: 'minioadmin', secretAccessKey: 'minio-secret-value' }, +} + +const KUBO: StorageProfile = { + name: 'kubo', + settings: { kind: 'ipfs-kubo', provider: 'kubo', api: 'http://127.0.0.1:5001', gateway: 'http://127.0.0.1:8081', publicGateway: '', pinningEndpoint: '' }, + secrets: {}, +} + +describe('profiles', () => { + it('accepts a sound S3 profile and names what is wrong otherwise', () => { + expect(profileProblem(S3)).toBeNull() + const s3 = S3.settings as Extract + const bad = (patch: Partial, secrets = S3.secrets): string | null => profileProblem({ ...S3, settings: { ...s3, ...patch }, secrets }) + expect(bad({ endpoint: 'https://x.example/bucket' })).toMatch(/origin/) + expect(bad({ endpoint: 'https://user:pw@x.example' })).toMatch(/user name/) + expect(bad({ endpoint: 'http://minio.example.org' })).toMatch(/https/) + expect(bad({ bucket: 'Forge' })).toMatch(/lowercase/) + expect(bad({ prefix: 'a/../b' })).toMatch(/segment/) + expect(bad({ publicUrl: '' })).toMatch(/public URL/) + expect(bad({ pathStyle: false, endpoint: 'https://1.2.3.4' })).toMatch(/path-style/) + expect(bad({}, {})).toMatch(/access key/) + expect(profileProblem({ ...S3, name: 'a,b' })).toMatch(/name/) + }) + + it('keys artifacts like the CLI', () => { + const s3 = S3.settings as Extract + expect(artifactKey(s3, 'ab'.repeat(32))).toBe(`web/packs/${'ab'.repeat(32)}.pack`) + expect(artifactKey({ ...s3, prefix: '' }, 'cd')).toBe('packs/cd.pack') + expect(artifactKey({ ...s3, prefix: '/a/b//' }, 'cd')).toBe('a/b/packs/cd.pack') + }) + + it('maps replication choices to policies and back', () => { + expect(policyFor(['a', 'b'], 'one')).toEqual({ targets: ['a', 'b'], replicas: 1, platformFallback: false }) + expect(policyFor(['a', 'b'], 'all')).toEqual({ targets: ['a', 'b'], replicas: 2, platformFallback: false }) + expect(policyFor(['a'], 'fallback')).toEqual({ targets: ['a'], replicas: 1, platformFallback: true }) + for (const c of ['one', 'all', 'fallback'] as const) expect(choiceOf(policyFor(['a', 'b'], c))).toBe(c) + }) +}) + +describe('the stored configuration', () => { + it('prunes a removed profile out of every policy', () => { + let c = withProfile(withProfile(EMPTY_STORAGE_CONFIG, S3), KUBO) + c = { ...c, defaultPolicy: policyFor(['minio', 'kubo'], 'all') } + c = withRepoPolicy(c, 'R1', policyFor(['kubo'], 'one')) + const after = withoutProfile(c, 'kubo') + expect(after.profiles.map((p) => p.name)).toEqual(['minio']) + expect(after.defaultPolicy).toEqual({ targets: ['minio'], replicas: 1, platformFallback: false }) + expect(after.repoPolicies).toEqual({}) + expect(policyForRepo(after, 'R1')).toEqual(after.defaultPolicy) + }) + + it('refuses a policy naming unknown profiles or too many copies', () => { + const c = withProfile(EMPTY_STORAGE_CONFIG, S3) + expect(policyProblem(c, policyFor(['minio'], 'one'))).toBeNull() + expect(policyProblem(c, policyFor(['nope'], 'one'))).toMatch(/no storage profile/) + expect(policyProblem(c, { targets: ['minio'], replicas: 2, platformFallback: false })).toMatch(/copies/) + expect(policyProblem(c, { targets: [], replicas: 1, platformFallback: false })).toMatch(/at least one/) + }) +}) + +describe('the configuration sealed in the vault', () => { + beforeEach(() => { + resetMemoryStores() + lockVault() + }) + + it('stores secrets encrypted, reads them back unlocked, and refuses when locked', async () => { + await storeInVault('devnet', { identityId: ID, keyId: 5, wif: 'cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy' }, { passphrase: 'correct horse battery' }) + const config = withRepoPolicy(withProfile(EMPTY_STORAGE_CONFIG, S3), 'R1', policyFor(['minio'], 'one')) + await saveStorageConfig('devnet', ID, config) + const dump = JSON.stringify(await idbEntries('vault'), (_k, v: unknown) => (v instanceof Uint8Array ? Array.from(v) : v)) + expect(dump).not.toContain('minio-secret-value') + expect(dump).not.toContain('forge-byo') + expect(await loadStorageConfig('devnet', ID)).toEqual(config) + lockVault() + await expect(loadStorageConfig('devnet', ID)).rejects.toThrow(/unlock/) + await unlockWithPassphrase('devnet', ID, 'correct horse battery') + expect((await loadStorageConfig('devnet', ID)).profiles[0]?.secrets.secretAccessKey).toBe('minio-secret-value') + }, 60_000) + + it('carries the storage settings across a key renewal', async () => { + const wif = 'cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy' + await storeInVault('devnet', { identityId: ID, keyId: 5, wif }, { passphrase: 'correct horse battery' }) + await saveStorageConfig('devnet', ID, withProfile(EMPTY_STORAGE_CONFIG, S3)) + await storeInVault('devnet', { identityId: ID, keyId: 6, wif }, { passphrase: 'a different passphrase' }) + lockVault() + await unlockWithPassphrase('devnet', ID, 'a different passphrase') + expect((await loadStorageConfig('devnet', ID)).profiles.map((p) => p.name)).toEqual(['minio']) + }, 90_000) +}) + +describe('chunks and manifests', () => { + it('splits exactly as forge-core pack::split', () => { + expect(splitChunks(new Uint8Array(0))).toEqual([]) + const bytes = new Uint8Array(CHUNK_PAYLOAD_MAX * 2 + FIELD_MAX + 7).map((_, i) => i % 251) + const chunks = splitChunks(bytes) + expect(chunks.map((c) => c.seq)).toEqual([0, 1, 2]) + expect(chunks.map((c) => c.fields.map((f) => f.length))).toEqual([[FIELD_MAX, FIELD_MAX, FIELD_MAX], [FIELD_MAX, FIELD_MAX, FIELD_MAX], [FIELD_MAX, 7]]) + const joined = new Uint8Array(chunks.flatMap((c) => c.fields.flatMap((f) => Array.from(f)))) + expect(joined).toEqual(bytes) + }) + + it('orders URIs platform, then http(s), then the rest, and fits the budget', () => { + expect(orderUris([['https://a/p', 's3://b/p'], ['ipfs://bafy', 'https://gw/ipfs/bafy'], ['platform://C/R/O/h']])).toEqual([ + 'platform://C/R/O/h', + 'https://a/p', + 'https://gw/ipfs/bafy', + 's3://b/p', + 'ipfs://bafy', + ]) + const many = Array.from({ length: 5 }, (_, i) => [`https://h${i}/p`, `s3://b${i}/p`]) + expect(fitManifestUris(orderUris(many))).toHaveLength(5) + expect(() => fitManifestUris([`https://x/${'a'.repeat(400)}`])).toThrow(/do not fit/) + expect(manifestUrisProblem([])).toMatch(/no confirmed copy/) + }) +}) + +describe('CORS fix blocks', () => { + it('are valid JSON naming the bucket, Range for reads and PUT for this origin', () => { + for (const p of ['r2', 'aws', 'b2'] as const) { + const fix = corsFix(p, 'my-bucket', 'https://forge.dashhq.org') + expect(fix.where + fix.text).toContain('my-bucket'.slice(0, p === 'r2' || p === 'aws' || p === 'b2' ? 9 : 0)) + const json: unknown = JSON.parse(fix.text) + const text = JSON.stringify(json).toLowerCase() + expect(text).toContain('range') + expect(text).toMatch(/put/) + expect(text).toContain('https://forge.dashhq.org') + expect(text).toContain('x-amz-content-sha256') + } + expect(corsFix('kubo', '', 'https://forge.dashhq.org').text).toContain('API.HTTPHeaders.Access-Control-Allow-Origin') + }) +}) + +// --------------------------------------------------------------------------- +// The replication engine against in-memory S3 and kubo +// --------------------------------------------------------------------------- + +const REPO: V2RepoRef = { kind: 'v2', forge: { core: 'CORE', collab: 'COLLAB', group: 'G' }, repoId: 'REPO', ownerId: ID, name: 'r', visibility: 'public' } +const AUTH: WriteAuth = { identityId: ID, network: 'devnet', getSigningKeyWif: () => '' } +const SDK = {} as EvoSDK + +function fakeNetwork(opts: { s3Down?: boolean; corruptOnce?: boolean } = {}) { + const objects = new Map() + let corrupt = opts.corruptOnce === true + const puts: string[] = [] + const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = new URL(String(input)) + const method = init?.method ?? 'GET' + if (url.port === '9000') { + if (opts.s3Down) throw new TypeError('Failed to fetch') + const key = url.pathname + if (method === 'PUT') { + const body = new Uint8Array(init?.body as ArrayBuffer | Uint8Array) + puts.push(key) + objects.set(key, new Uint8Array(corrupt ? body.map((b) => b ^ 1) : body)) + corrupt = false + return new Response(null, { status: 200 }) + } + const obj = objects.get(key) + if (!obj) return new Response(null, { status: 404 }) + if (method === 'HEAD') return new Response(null, { status: 200, headers: { 'content-length': String(obj.length) } }) + return new Response(new Uint8Array(obj), { status: 200 }) + } + if (url.port === '5001') { + if (url.pathname.endsWith('/add')) { + const file = (init?.body as FormData).get('file') as Blob + const bytes = new Uint8Array(await file.arrayBuffer()) + const cid = cidV1RawLeaves(bytes) + objects.set(`ipfs:${cid}`, bytes) + return new Response(JSON.stringify({ Hash: cid })) + } + if (url.pathname.endsWith('/pin/ls')) { + const cid = url.searchParams.get('arg') ?? '' + return objects.has(`ipfs:${cid}`) ? new Response(JSON.stringify({ Keys: { [cid]: { Type: 'recursive' } } })) : new Response('{"Message":"not pinned"}', { status: 500 }) + } + return new Response('{}') + } + if (url.port === '8081') { + const obj = objects.get(`ipfs:${url.pathname.split('/').pop() ?? ''}`) + return obj ? new Response(new Uint8Array(obj)) : new Response(null, { status: 404 }) + } + throw new Error(`unexpected ${url}`) + }) + return { fetchMock, objects, puts } +} + +describe('storeArtifact', () => { + const bytes = new TextEncoder().encode('PACK a small pack') + afterEach(() => vi.unstubAllGlobals()) + + it('stores on S3, verifies by re-read, and records the public URL before the s3:// locator', async () => { + const net = fakeNetwork() + vi.stubGlobal('fetch', net.fetchMock) + const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { + policy: policyFor(['minio'], 'one'), + profiles: [S3], + confirmPlatform: async () => false, + }) + const hash = await sha256Hex(bytes) + expect(stored).toMatchObject({ packHash: hash, storage: 1, chunkCount: 0, confirmed: ['minio'], failures: [] }) + expect(stored.uris).toEqual([`http://127.0.0.1:9000/forge-byo/web/packs/${hash}.pack`, `s3://forge-byo/web/packs/${hash}.pack`]) + // The request that wrote it was signed. + const put = net.fetchMock.mock.calls.find(([, init]) => init?.method === 'PUT') + expect(new Headers(put?.[1]?.headers).get('authorization')).toMatch(/^AWS4-HMAC-SHA256 Credential=minioadmin\//) + expect(new Headers(put?.[1]?.headers).get('x-amz-content-sha256')).toBe(hash) + }) + + it('re-uploads once when the stored copy does not verify', async () => { + const net = fakeNetwork({ corruptOnce: true }) + vi.stubGlobal('fetch', net.fetchMock) + const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['minio'], 'one'), profiles: [S3], confirmPlatform: async () => false }) + expect(stored.confirmed).toEqual(['minio']) + expect(net.puts).toHaveLength(2) + }) + + it('fails the policy (writing nothing to Platform) when too few targets confirm', async () => { + vi.stubGlobal('fetch', fakeNetwork({ s3Down: true }).fetchMock) + const confirm = vi.fn(async () => true) + const run = storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['minio', 'kubo'], 'all'), profiles: [S3, KUBO], confirmPlatform: confirm }) + await expect(run).rejects.toBeInstanceOf(ReplicationError) + await expect(run).rejects.toThrow(/1 of 2 required .*minio: signed PUT.*Nothing was written to Platform/) + expect(confirm).not.toHaveBeenCalled() + }) + + it('verifies the kubo CID against the local derivation', async () => { + vi.stubGlobal('fetch', fakeNetwork().fetchMock) + const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['kubo'], 'one'), profiles: [{ ...KUBO, settings: { ...KUBO.settings, gateway: '' } as ProfilePublic }], confirmPlatform: async () => false }) + expect(stored.uris).toEqual([`ipfs://${cidV1RawLeaves(bytes)}`]) + }) + + it('asks before storing on Platform when nothing is configured, and stops when declined', async () => { + const confirm = vi.fn(async () => false) + await expect(storeArtifact(SDK, AUTH, REPO, bytes, { policy: null, profiles: [], confirmPlatform: confirm })).rejects.toBeInstanceOf(PlatformDeclinedError) + expect(confirm).toHaveBeenCalledWith(expect.objectContaining({ bytes: bytes.length, estimateCredits: expect.any(Number) })) + }) +}) diff --git a/forge-web/lib/storage/store.ts b/forge-web/lib/storage/store.ts new file mode 100644 index 000000000..96734fd2a --- /dev/null +++ b/forge-web/lib/storage/store.ts @@ -0,0 +1,124 @@ +/** + * The browser's storage configuration: named profiles, the default browser-push policy, and + * per-repo overrides. All of it lives in ONE blob sealed in the vault (`lib/auth/vault.ts` + * `writeStorageBlob`): the profiles' secrets must never touch localStorage or plaintext + * IndexedDB, and keeping the non-secret parts next to them means there is exactly one place + * to read, one to forget, and nothing to reconcile. Reading needs an unlocked session, which + * every push needs anyway. + * + * What is stored is parsed with zod on the way out: a tampered or foreign blob fails to + * decrypt (AES-GCM with the network and identity as additional data) and, if it did not, + * would still not reach the uploader unvalidated. + */ + +import { z } from 'zod' + +import type { Network } from '../constants' +import { readStorageBlob, writeStorageBlob } from '../auth/vault' +import { + policySchema, + profilePublicSchema, + profileSecretsSchema, + type StoragePolicy, + type StorageProfile, +} from './profiles' + +/** Everything this browser knows about the user's storage. */ +export interface StorageConfig { + readonly version: 1 + readonly profiles: readonly StorageProfile[] + /** The policy browser pushes use when a repo has no override. */ + readonly defaultPolicy: StoragePolicy | null + /** repoId → the policy for browser pushes to that repo. */ + readonly repoPolicies: Readonly> + /** Profile name → the last browser test: when, and whether every row passed. */ + readonly lastTests: Readonly> +} + +export const EMPTY_STORAGE_CONFIG: StorageConfig = { + version: 1, + profiles: [], + defaultPolicy: null, + repoPolicies: {}, + lastTests: {}, +} + +const configSchema = z.object({ + version: z.literal(1), + profiles: z + .array(z.object({ name: z.string().max(64), settings: profilePublicSchema, secrets: profileSecretsSchema })) + .max(32), + defaultPolicy: policySchema.nullable(), + repoPolicies: z.record(z.string().max(64), policySchema), + lastTests: z.record(z.string().max(64), z.object({ at: z.number(), ok: z.boolean() })), +}) + +/** The stored configuration (empty when none is stored). Needs the vault unlocked. */ +export async function loadStorageConfig(network: Network, identityId: string): Promise { + const raw = await readStorageBlob(network, identityId) + if (raw === null) return EMPTY_STORAGE_CONFIG + const parsed = configSchema.safeParse(raw) + if (!parsed.success) throw new Error('the stored storage settings are not in a shape this app understands') + return parsed.data as StorageConfig +} + +/** Seal `config` in the vault. Needs the vault unlocked. */ +export async function saveStorageConfig(network: Network, identityId: string, config: StorageConfig): Promise { + const parsed = configSchema.parse(config) + await writeStorageBlob(network, identityId, parsed) +} + +/** `config` with `profile` added or replaced (by name). */ +export function withProfile(config: StorageConfig, profile: StorageProfile): StorageConfig { + const profiles = [...config.profiles.filter((p) => p.name !== profile.name), profile].sort((a, b) => a.name.localeCompare(b.name)) + return { ...config, profiles } +} + +/** + * `config` without the profile `name`: it also leaves every policy (a policy naming a profile + * that no longer exists would fail every push). A policy left with no targets is removed. + */ +export function withoutProfile(config: StorageConfig, name: string): StorageConfig { + const prune = (p: StoragePolicy | null): StoragePolicy | null => { + if (p === null) return null + const targets = p.targets.filter((t) => t !== name) + if (targets.length === 0) return null + return { ...p, targets, replicas: Math.min(p.replicas, targets.length) } + } + const repoPolicies: Record = {} + for (const [repo, p] of Object.entries(config.repoPolicies)) { + const next = prune(p) + if (next) repoPolicies[repo] = next + } + const lastTests = { ...config.lastTests } + delete lastTests[name] + return { + ...config, + profiles: config.profiles.filter((p) => p.name !== name), + defaultPolicy: prune(config.defaultPolicy), + repoPolicies, + lastTests, + } +} + +/** The policy browser pushes to `repoId` use: its override, else the default, else none. */ +export function policyForRepo(config: StorageConfig, repoId: string): StoragePolicy | null { + return config.repoPolicies[repoId] ?? config.defaultPolicy +} + +/** `config` with `repoId`'s override set (or cleared with null). */ +export function withRepoPolicy(config: StorageConfig, repoId: string, policy: StoragePolicy | null): StorageConfig { + const repoPolicies = { ...config.repoPolicies } + if (policy === null) delete repoPolicies[repoId] + else repoPolicies[repoId] = policy + return { ...config, repoPolicies } +} + +/** Why a policy cannot be used with `config`'s profiles, or null. */ +export function policyProblem(config: StorageConfig, policy: StoragePolicy): string | null { + if (policy.targets.length === 0) return 'choose at least one place to store packs' + const missing = policy.targets.filter((t) => !config.profiles.some((p) => p.name === t)) + if (missing.length > 0) return `no storage profile named ${missing.join(', ')}` + if (policy.replicas > policy.targets.length) return `${policy.replicas} copies are required but only ${policy.targets.length} places are chosen` + return null +} diff --git a/forge-web/lib/storage/upload.live.test.ts b/forge-web/lib/storage/upload.live.test.ts new file mode 100644 index 000000000..f3b95d5fd --- /dev/null +++ b/forge-web/lib/storage/upload.live.test.ts @@ -0,0 +1,97 @@ +/** + * Live browser-upload path — SKIPPED by default (devnet writes + a local MinIO). + * + * docker compose -f infra/docker-compose.yml up -d minio minio-init + * FORGE_LIVE=1 NEXT_PUBLIC_NETWORK=devnet NEXT_PUBLIC_DEVNET_NAME=moutai \ + * pnpm exec vitest run lib/storage/upload.live.test.ts + * + * As the moutai MAINTAINER (a maintainer of its own `forge-v2-empty`, the fixture no push has + * touched): store a small artifact in the `forge-byo` bucket with SigV4 (the same code the + * browser runs, on Node's WebCrypto + fetch), verified by re-read; write its `packManifest` + * (kind 2, a flatIndex slot nothing reads for browsing); then read the manifest back from + * Platform and fetch + hash the bytes through the web reader (`loadArtifactBytesProgress`), + * as any browser would. A rerun is idempotent (content-addressed key, the signer's own + * manifest slot). Spend: one manifest document. + */ + +import { readFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' + +import { describe, expect, it } from 'vitest' + +import { DEFAULT_NETWORK, NETWORKS } from '../constants' +import { evoSdkService, type WriteAuth } from '../sdk' +import { parseIdentityFileText } from '../auth' +import { readV2PackCopies } from '../repo' +import { loadArtifactBytesProgress } from '../view/browse-source' +import { retryWhileMissing } from '../view/retry' +import type { V2RepoRef } from '../repo/contract' +import { storeAndRecordPack, policyFor, type StorageProfile } from './index' +import { sha256Hex } from './sigv4' + +const LIVE = process.env['FORGE_LIVE'] === '1' && DEFAULT_NETWORK === 'devnet' +const MAINTAINER_FILE = join(homedir(), '.config/dash-forge/test-identities/devnet-moutai/MAINTAINER.identity.json') +const EMPTY_REPO = 'F9puk5NBQyySFbgj9yUfubdXAVv1YV7Zk2KDvfyHzxLU' + +const MINIO: StorageProfile = { + name: 'minio', + settings: { + kind: 's3', + provider: 'minio', + endpoint: 'http://127.0.0.1:9000', + region: 'us-east-1', + bucket: 'forge-byo', + pathStyle: true, + publicUrl: 'http://127.0.0.1:9000/forge-byo', + prefix: 'web-live', + }, + secrets: { accessKeyId: 'minioadmin', secretAccessKey: 'minioadmin' }, +} + +describe.skipIf(!LIVE)('live browser upload to MinIO + manifest on moutai', () => { + it( + 'stores, verifies, records, and reads back through the web reader', + async () => { + const parsed = parseIdentityFileText(readFileSync(MAINTAINER_FILE, 'utf8')) + const auth: WriteAuth = { identityId: parsed.identityId, network: 'devnet', getSigningKeyWif: () => parsed.signingKeyWif } + const v2 = NETWORKS.devnet.v2 + if (!v2) throw new Error('no forge-v2 deployment on this devnet') + await evoSdkService.initialize({ network: 'devnet', contractIds: [v2.core, v2.collab], timeoutMs: 20000 }) + const sdk = evoSdkService.getSdk() + const repo: V2RepoRef = { kind: 'v2', forge: v2, repoId: EMPTY_REPO, ownerId: parsed.identityId, name: 'forge-v2-empty', visibility: 'public' } + + const bytes = new TextEncoder().encode(`forge-web live upload artifact ${EMPTY_REPO}\n`) + const hash = await sha256Hex(bytes) + const events: string[] = [] + const { stored, manifest } = await storeAndRecordPack( + sdk, + auth, + repo, + bytes, + { kind: 2, objectCount: 0 }, + { + policy: policyFor(['minio'], 'one'), + profiles: [MINIO], + confirmPlatform: async () => false, + onStep: (e) => events.push(`${e.target}:${e.phase}`), + intent: `live-upload:${hash}`, + }, + ) + // eslint-disable-next-line no-console + console.log('stored', { uris: stored.uris, manifest: manifest.documentId, cost: manifest.cost.dash, events }) + expect(stored.storage).toBe(1) + expect(stored.uris[0]).toBe(`http://127.0.0.1:9000/forge-byo/web-live/packs/${hash}.pack`) + expect(stored.uris[1]).toBe(`s3://forge-byo/web-live/packs/${hash}.pack`) + expect(events).toEqual(['minio:start', 'minio:done']) + + // The node answering the read may be a block behind the one that confirmed the write. + const copy = await retryWhileMissing(() => readV2PackCopies(sdk, repo, hash, 2), 8) + expect(copy, 'the manifest should read back from Platform').not.toBeNull() + expect(copy?.uris).toEqual(stored.uris) + const got = await loadArtifactBytesProgress(sdk, repo, copy!) + expect(await sha256Hex(got)).toBe(hash) + }, + 180_000, + ) +}) diff --git a/forge-web/lib/storage/upload.ts b/forge-web/lib/storage/upload.ts new file mode 100644 index 000000000..dbcbb718a --- /dev/null +++ b/forge-web/lib/storage/upload.ts @@ -0,0 +1,276 @@ +/** + * Browser uploads of pack artifacts under the user's storage policy: the web counterpart of + * forge-core `storage/targets.rs` (`replicate`, `ExternalTarget`) and `StoredArtifact`. + * + * Order of events, the same guarantee the CLI gives: the bytes are stored AND verified on at + * least `replicas` targets before this resolves, and only then may the caller write the + * `packManifest` (and after it, refs). Nothing on Platform is written here unless Platform is + * itself the target (or the fallback the user confirmed, with its price). + * + * Verification is by re-read, as in the CLI: S3 — HEAD, then the whole object up to 16 MiB + * re-hashed (larger: size plus byte-exact head and tail windows); IPFS — kubo's CID must equal + * the local derivation and be pinned, then a gateway re-read when one is configured. + * S3 object keys are content-addressed (`packs/.pack`), so a re-upload is + * idempotent and a same-size object at the key is re-verified rather than trusted. + */ + +import type { EvoSDK } from '@dashevo/evo-sdk' + +import type { WriteAuth } from '../sdk' +import { estimateChunkCredits } from '../sdk/cost' +import type { V2RepoRef } from '../repo/contract' +import { putPlatformChunks } from '../repo/push' +import { addVerified, gatewayUrl, remotePin } from './ipfs' +import { artifactKey, type StoragePolicy, type StorageProfile } from './profiles' +import { getObject, headObject, publicObjectUrl, putObject, s3Uri, type S3Settings } from './s3' +import { sha256Hex } from './sigv4' + +/** Artifacts up to this size are verified by a full re-download and SHA-256. */ +export const FULL_VERIFY_MAX = 16 * 1024 * 1024 +const EDGE_WINDOW = 64 * 1024 + +/** A target that did not confirm. */ +export interface TargetFailure { + readonly target: string + readonly reason: string +} + +/** Progress of one target. */ +export type UploadEvent = + | { readonly target: string; readonly phase: 'start' } + | { readonly target: string; readonly phase: 'progress'; readonly done: number; readonly total: number } + | { readonly target: string; readonly phase: 'done'; readonly uris: readonly string[] } + | { readonly target: string; readonly phase: 'failed'; readonly reason: string } + +/** Where an artifact now lives: what its `packManifest` records. */ +export interface StoredArtifact { + readonly packHash: string + readonly sizeBytes: number + /** 0 when a Platform copy exists (chunks), else 1. */ + readonly storage: 0 | 1 + readonly chunkCount: number + /** `platform://` first, then public http(s) URLs, then the rest (`ipfs://`, `s3://`). */ + readonly uris: readonly string[] + /** Targets that confirmed, in policy order. */ + readonly confirmed: readonly string[] + readonly failures: readonly TargetFailure[] +} + +/** The price of putting `bytes` on Platform, asked before paying it. */ +export interface PlatformQuestion { + readonly bytes: number + readonly estimateCredits: number + /** Why Platform is being offered: no storage configured, or the policy was not met. */ + readonly reason: string +} + +/** Fewer than the required targets confirmed. */ +export class ReplicationError extends Error { + constructor( + readonly required: number, + readonly confirmed: readonly string[], + readonly failures: readonly TargetFailure[], + ) { + super( + `storage policy not met: ${confirmed.length} of ${required} required target(s) confirmed` + + failures.map((f) => `; ${f.target}: ${f.reason}`).join('') + + '. Nothing was written to Platform.', + ) + this.name = 'ReplicationError' + } +} + +/** The user declined to store on Platform. */ +export class PlatformDeclinedError extends Error { + constructor() { + super('Nothing was stored: storing on Platform was declined. Configure storage in Settings → Storage and try again.') + this.name = 'PlatformDeclinedError' + } +} + +function equal(a: Uint8Array, b: Uint8Array): boolean { + return a.length === b.length && a.every((x, i) => x === b[i]) +} + +/** Re-read an S3 copy and check it is exactly `bytes`. */ +async function verifyS3(s: S3Settings, p: StorageProfile, key: string, bytes: Uint8Array, hashHex: string): Promise { + if (bytes.length <= FULL_VERIFY_MAX) { + const got = await getObject(s, p.secrets, key) + if ((await sha256Hex(got)) !== hashHex) throw new Error(`re-read of ${key} returned ${got.length} bytes that do not hash to the artifact`) + return + } + const size = await headObject(s, p.secrets, key) + if (size !== bytes.length) throw new Error(`re-read of ${key}: object missing or wrong size (${size} ≠ ${bytes.length})`) + for (const [start, end] of [ + [0, Math.min(EDGE_WINDOW, bytes.length)], + [Math.max(0, bytes.length - EDGE_WINDOW), bytes.length], + ] as const) { + const got = await getObject(s, p.secrets, key, `bytes=${start}-${end - 1}`) + if (!equal(got, bytes.subarray(start, end))) throw new Error(`re-read of ${key} bytes ${start}..${end} differ from the artifact`) + } +} + +/** Store and verify on one external profile; the URIs its manifest entry records. */ +async function storeExternal(p: StorageProfile, bytes: Uint8Array, hashHex: string): Promise { + const s = p.settings + if (s.kind === 's3') { + const key = artifactKey(s, hashHex) + const existing = await headObject(s, p.secrets, key).catch(() => null) + if (existing !== bytes.length) await putObject(s, p.secrets, key, bytes) + try { + await verifyS3(s, p, key, bytes, hashHex) + } catch (first) { + // A same-size but corrupt object at a content-addressed key would fail every push: + // re-upload once, unconditionally, and check again. + await putObject(s, p.secrets, key, bytes) + try { + await verifyS3(s, p, key, bytes, hashHex) + } catch (second) { + throw new Error(`${msg(second)} (also after a re-upload; first attempt: ${msg(first)})`) + } + } + return [publicObjectUrl(s, key), s3Uri(s, key)] + } + if (s.kind === 'ipfs-kubo' || s.kind === 'ipfs-pinning-service') { + const cid = await addVerified(s, p.secrets, bytes) + if (s.kind === 'ipfs-pinning-service') await remotePin(s, p.secrets, cid, `dash-forge ${hashHex.slice(0, 16)}`) + if (s.gateway !== '') { + const resp = await fetch(gatewayUrl(s.gateway, cid), { credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(60_000) }) + const got = new Uint8Array(await resp.arrayBuffer()) + if (!resp.ok || (await sha256Hex(got)) !== hashHex) throw new Error(`re-read of ipfs://${cid} through the gateway did not return the artifact`) + } + const uris = [`ipfs://${cid}`] + if (s.publicGateway !== '') uris.unshift(gatewayUrl(s.publicGateway, cid)) + return uris + } + throw new Error('not an external profile') +} + +function msg(e: unknown): string { + return e instanceof Error ? e.message : String(e) +} + +/** + * Every recorded URI, de-duplicated: the `platform://` locator first (released helpers read + * `uris[0]` of a storage-0 manifest as the chunk locator), then public http(s) URLs, then the + * rest — each group in policy order (parity with forge-core `Replication::uris`). + */ +export function orderUris(groups: readonly (readonly string[])[]): string[] { + const buckets: string[][] = [[], [], []] + for (const uris of groups) { + for (const u of uris) { + const g = u.startsWith('platform://') ? 0 : /^https?:\/\//.test(u) ? 1 : 2 + const bucket = buckets[g] as string[] + if (!bucket.includes(u)) bucket.push(u) + } + } + return buckets.flat() +} + +/** + * Trim to the manifest's `uris` budget (8 × 300 bytes): private `s3://` locators go first + * (readers without that profile cannot use them); then an error, never a silent truncation. + */ +export function fitManifestUris(uris: readonly string[]): string[] { + const fits = (list: readonly string[]): boolean => list.length > 0 && list.length <= 8 && list.every((u) => new TextEncoder().encode(u).length <= 300) + if (fits(uris)) return [...uris] + const noS3 = uris.filter((u) => !u.startsWith('s3://')) + if (fits(noS3)) return noS3 + throw new Error("the confirmed copies' URIs do not fit a manifest (at most 8, each up to 300 bytes); use shorter public URLs or fewer targets") +} + +/** + * Store `bytes` under `policy`, verifying every copy, and return what the manifest records. + * With no policy, or when the policy is not met and it allows the Platform fallback, the user + * is asked (with the price) before any chunk is written; declining throws + * {@link PlatformDeclinedError}. Anything else short of `replicas` throws {@link ReplicationError}. + */ +export async function storeArtifact( + sdk: EvoSDK, + auth: WriteAuth, + repo: V2RepoRef, + bytes: Uint8Array, + opts: { + readonly policy: StoragePolicy | null + readonly profiles: readonly StorageProfile[] + readonly confirmPlatform: (q: PlatformQuestion) => Promise + readonly onStep?: (e: UploadEvent) => void + }, +): Promise { + const hashHex = await sha256Hex(bytes) + const step = opts.onStep ?? (() => undefined) + const confirmed: { target: string; uris: string[]; platform: boolean }[] = [] + const failures: TargetFailure[] = [] + let chunkCount = 0 + + const platformCopy = async (target: string): Promise => { + step({ target, phase: 'start' }) + try { + const r = await putPlatformChunks(sdk, auth, repo, bytes, hashHex, (done, total) => step({ target, phase: 'progress', done, total })) + chunkCount = r.chunkCount + confirmed.push({ target, uris: [r.locator], platform: true }) + step({ target, phase: 'done', uris: [r.locator] }) + } catch (e) { + failures.push({ target, reason: msg(e) }) + step({ target, phase: 'failed', reason: msg(e) }) + throw e + } + } + + const policy = opts.policy + if (policy === null || policy.targets.length === 0) { + const ok = await opts.confirmPlatform({ bytes: bytes.length, estimateCredits: estimateChunkCredits(bytes.length), reason: 'No storage is configured for browser pushes to this repo.' }) + if (!ok) throw new PlatformDeclinedError() + await platformCopy('platform') + } else { + const byName = new Map(opts.profiles.map((p) => [p.name, p])) + const targets = policy.targets.map((name) => ({ name, profile: byName.get(name) })) + const external = targets.filter((t) => t.profile?.settings.kind !== 'platform') + const onChain = targets.filter((t) => t.profile?.settings.kind === 'platform') + await Promise.all( + external.map(async ({ name, profile }) => { + step({ target: name, phase: 'start' }) + try { + if (!profile) throw new Error('no such storage profile in this browser') + const uris = await storeExternal(profile, bytes, hashHex) + confirmed.push({ target: name, uris, platform: false }) + step({ target: name, phase: 'done', uris }) + } catch (e) { + failures.push({ target: name, reason: msg(e) }) + step({ target: name, phase: 'failed', reason: msg(e) }) + } + }), + ) + // Platform targets only if they can still make up the policy: never pay for chunks + // whose push is about to fail anyway. + for (let i = 0; i < onChain.length; i++) { + if (confirmed.length + (onChain.length - i) < policy.replicas) break + const t = onChain[i] as { name: string } + await platformCopy(t.name).catch(() => undefined) + } + if (confirmed.length < policy.replicas) { + if (!policy.platformFallback || confirmed.some((c) => c.platform)) throw new ReplicationError(policy.replicas, confirmed.map((c) => c.target), failures) + const ok = await opts.confirmPlatform({ + bytes: bytes.length, + estimateCredits: estimateChunkCredits(bytes.length), + reason: `Your storage did not confirm (${failures.map((f) => f.target).join(', ')}).`, + }) + if (!ok) throw new ReplicationError(policy.replicas, confirmed.map((c) => c.target), failures) + await platformCopy('platform (fallback)') + } + } + + // Keep policy order within each URI group. + const order = policy?.targets ?? [] + confirmed.sort((a, b) => order.indexOf(a.target) - order.indexOf(b.target)) + const hasPlatform = confirmed.some((c) => c.platform) + return { + packHash: hashHex, + sizeBytes: bytes.length, + storage: hasPlatform ? 0 : 1, + chunkCount: hasPlatform ? chunkCount : 0, + uris: fitManifestUris(orderUris(confirmed.map((c) => c.uris))), + confirmed: confirmed.map((c) => c.target), + failures, + } +} diff --git a/forge-web/tailwind.config.js b/forge-web/tailwind.config.js index 60b0ebc89..491713f20 100644 --- a/forge-web/tailwind.config.js +++ b/forge-web/tailwind.config.js @@ -51,7 +51,11 @@ module.exports = { DEFAULT: '#d97706', // degraded availability — icons, borders, tints; text on dark 700: '#b45309', // TEXT on light surfaces (5.02:1 on white; the base value is 3.19:1) }, - danger: '#dc2626', // force-push, delete, failed verification + danger: { + DEFAULT: '#dc2626', // force-push, delete, failed verification: icons, borders, tints, text on light + 400: '#f87171', // TEXT on dark surfaces (the base value is 4.01:1 on anvil-950) + 700: '#b91c1c', // TEXT on light surfaces (the base value is 4.4:1 on anvil-100) + }, // Dash brand blue — identity/credits/network UI only. The brand value is for fills, // tints and icons; it is under 4.5:1 as TEXT on every surface in both themes, so // text uses the WCAG AA shades: `text-dash-600 dark:text-dash-400` (the same From 56b9d58510ece4818204f226968d0dfbae547523 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 26 Sep 2026 16:44:59 -0500 Subject: [PATCH 2/5] refactor(web): simplify the storage upload and probe code Shared byte and error helpers, one StoreOptions type, a status-hint switch, and fewer duplicated failure paths; no behavior change. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/components/storage/profile-form.tsx | 19 ++-- .../storage/repo-storage-policy.tsx | 3 +- forge-web/components/storage/storage-test.tsx | 3 +- .../components/storage/storage-wizard.tsx | 16 +-- forge-web/lib/auth/vault.ts | 2 +- forge-web/lib/repo/push.ts | 9 +- forge-web/lib/storage/cors.ts | 9 +- forge-web/lib/storage/index.ts | 37 ++----- forge-web/lib/storage/ipfs.ts | 16 +-- forge-web/lib/storage/probe.ts | 98 ++++++++----------- forge-web/lib/storage/s3.ts | 29 +++--- forge-web/lib/storage/upload.ts | 63 ++++++------ forge-web/lib/storage/util.ts | 11 +++ 13 files changed, 154 insertions(+), 161 deletions(-) create mode 100644 forge-web/lib/storage/util.ts diff --git a/forge-web/components/storage/profile-form.tsx b/forge-web/components/storage/profile-form.tsx index f18844eac..46009ee8d 100644 --- a/forge-web/components/storage/profile-form.tsx +++ b/forge-web/components/storage/profile-form.tsx @@ -22,6 +22,9 @@ const SECRET_NOTE = 'Stored encrypted in this browser only. Never sent to Forge type Values = Record +/** Providers whose suggested profile name is the provider id itself (the rest get `-main`). */ +const BARE_NAMES: readonly ProviderId[] = ['aws', 'minio', 'kubo', 'pinning'] + function initialValues(provider: ProviderId, existing: StorageProfile | null): Values { const preset = providerPreset(provider) if (existing) { @@ -31,7 +34,7 @@ function initialValues(provider: ProviderId, existing: StorageProfile | null): V } return out } - const base: Values = { name: provider === 'aws' ? 'aws' : provider === 'minio' ? 'minio' : provider === 'kubo' ? 'kubo' : provider === 'pinning' ? 'pinning' : `${provider}-main` } + const base: Values = { name: BARE_NAMES.includes(provider) ? provider : `${provider}-main` } const empty: Values = preset.kind === 's3' ? { endpoint: '', region: 'us-east-1', bucket: '', pathStyle: true, publicUrl: '', prefix: '', accessKeyId: '', secretAccessKey: '', sessionToken: '' } @@ -42,6 +45,8 @@ function initialValues(provider: ProviderId, existing: StorageProfile | null): V } const str = (v: string | boolean | undefined): string => (typeof v === 'string' ? v.trim() : '') +/** A URL field, trimmed and without trailing slashes. */ +const url = (v: string | boolean | undefined): string => str(v).replace(/\/+$/, '') /** The profile the form's values describe. */ export function profileFromValues(provider: ProviderId, v: Values): StorageProfile { @@ -52,11 +57,11 @@ export function profileFromValues(provider: ProviderId, v: Values): StorageProfi settings = { kind: 's3', provider: provider as 'r2' | 'b2' | 'aws' | 'minio', - endpoint: str(v['endpoint']).replace(/\/+$/, ''), + endpoint: url(v['endpoint']), region: str(v['region']) || 'us-east-1', bucket: str(v['bucket']), pathStyle: v['pathStyle'] !== false, - publicUrl: str(v['publicUrl']).replace(/\/+$/, ''), + publicUrl: url(v['publicUrl']), prefix: str(v['prefix']), } return { name: str(v['name']), settings, secrets: { ...secret('accessKeyId'), ...secret('secretAccessKey'), ...secret('sessionToken') } } @@ -65,10 +70,10 @@ export function profileFromValues(provider: ProviderId, v: Values): StorageProfi settings = { kind: preset.kind === 'ipfs-pinning-service' ? 'ipfs-pinning-service' : 'ipfs-kubo', provider: provider as 'kubo' | 'pinning', - api: str(v['api']).replace(/\/+$/, ''), - gateway: str(v['gateway']).replace(/\/+$/, ''), - publicGateway: str(v['publicGateway']).replace(/\/+$/, ''), - pinningEndpoint: str(v['pinningEndpoint']).replace(/\/+$/, ''), + api: url(v['api']), + gateway: url(v['gateway']), + publicGateway: url(v['publicGateway']), + pinningEndpoint: url(v['pinningEndpoint']), } return { name: str(v['name']), settings, secrets: { ...secret('apiAuth'), ...secret('pinningToken') } } } diff --git a/forge-web/components/storage/repo-storage-policy.tsx b/forge-web/components/storage/repo-storage-policy.tsx index 72299978f..02504f4b0 100644 --- a/forge-web/components/storage/repo-storage-policy.tsx +++ b/forge-web/components/storage/repo-storage-policy.tsx @@ -11,6 +11,7 @@ import { useState } from 'react' import Link from 'next/link' import { HardDrive } from 'lucide-react' import { choiceOf, policyFor, policyForRepo, policyProblem, withRepoPolicy, type ReplicationChoice } from '@/lib/storage' +import { errText } from '@/lib/storage/util' import { useStorageConfig } from '@/hooks/use-storage-config' import { Button } from '@/components/ui/button' @@ -94,7 +95,7 @@ export function RepoStoragePolicy({ repoId }: { repoId: string }): JSX.Element { await save(withRepoPolicy(config, repoId, policy)) setMsg('Saved for this repo.') } catch (e) { - setMsg(e instanceof Error ? e.message : String(e)) + setMsg(errText(e)) } }} > diff --git a/forge-web/components/storage/storage-test.tsx b/forge-web/components/storage/storage-test.tsx index 435b731e4..6c1fa9d8a 100644 --- a/forge-web/components/storage/storage-test.tsx +++ b/forge-web/components/storage/storage-test.tsx @@ -18,6 +18,7 @@ import { type RowState, type StorageProfile, } from '@/lib/storage' +import { errText } from '@/lib/storage/util' import { Button } from '@/components/ui/button' import { CopyBlock } from '@/components/storage/copy-block' import { cn } from '@/lib/utils' @@ -51,7 +52,7 @@ export function StorageTest({ profile, onDone }: { profile: StorageProfile; onDo onDone(await probeProfile(profile, report)) } catch (e) { onDone(false) - setRows((prev) => (prev ?? []).map((r) => (r.state === 'running' || r.state === 'pending' ? { ...r, state: 'fail', detail: e instanceof Error ? e.message : String(e) } : r))) + setRows((prev) => (prev ?? []).map((r) => (r.state === 'running' || r.state === 'pending' ? { ...r, state: 'fail', detail: errText(e) } : r))) } finally { setRunning(false) } diff --git a/forge-web/components/storage/storage-wizard.tsx b/forge-web/components/storage/storage-wizard.tsx index 3cf9646e8..35aa9c6ad 100644 --- a/forge-web/components/storage/storage-wizard.tsx +++ b/forge-web/components/storage/storage-wizard.tsx @@ -15,6 +15,7 @@ import { choiceOf, policyFor, policyProblem, + providerPreset, withProfile, withoutProfile, type ProviderId, @@ -22,6 +23,7 @@ import { type StorageConfig, type StorageProfile, } from '@/lib/storage' +import { errText } from '@/lib/storage/util' import { useStorageConfig } from '@/hooks/use-storage-config' import { Button } from '@/components/ui/button' import { ErrorState, LoadingBlock } from '@/components/ui/states' @@ -136,7 +138,7 @@ function AddProfile({ save: (c: StorageConfig) => Promise onDone: () => void }): JSX.Element { - const preset = PROVIDERS.find((p) => p.id === provider) + const preset = providerPreset(provider) const [draft, setDraft] = useState<{ profile: StorageProfile; problem: string | null } | null>(null) const [tested, setTested] = useState(null) const [saving, setSaving] = useState(false) @@ -162,7 +164,7 @@ function AddProfile({ await save(withProfile(config, { name: 'platform', settings: { kind: 'platform', provider: 'platform' }, secrets: {} })) onDone() } catch (e) { - setSaveError(e instanceof Error ? e.message : String(e)) + setSaveError(errText(e)) } finally { setSaving(false) } @@ -182,10 +184,10 @@ function AddProfile({ const blocking = problem ?? (nameTaken ? 'a profile with that name exists' : null) return ( -
+
-

{existing ? `Edit ${existing.name}` : preset?.title}

- {preset?.blurb} +

{existing ? `Edit ${existing.name}` : preset.title}

+ {preset.blurb}
diff --git a/forge-web/lib/auth/vault.ts b/forge-web/lib/auth/vault.ts index c533be5af..d78e692d7 100644 --- a/forge-web/lib/auth/vault.ts +++ b/forge-web/lib/auth/vault.ts @@ -131,7 +131,7 @@ interface StorageBlob { async function deriveStorageKey(dataKey: Uint8Array, network: Network, identityId: string): Promise { const raw = hkdf(sha256, dataKey, enc.encode('dash-forge vault storage v1'), enc.encode(`${network}|${identityId}`), 32) try { - return await crypto.subtle.importKey('raw', buf(raw), { name: 'AES-GCM' }, false, ['encrypt', 'decrypt']) + return await aesKey(raw, ['encrypt', 'decrypt']) } finally { raw.fill(0) } diff --git a/forge-web/lib/repo/push.ts b/forge-web/lib/repo/push.ts index 7798113fa..bd9e1a1eb 100644 --- a/forge-web/lib/repo/push.ts +++ b/forge-web/lib/repo/push.ts @@ -66,6 +66,11 @@ function concatHex(list: readonly string[], width: number): Uint8Array { return out } +/** The result for a manifest this signer already recorded (nothing spent). */ +function alreadyRecorded(documentId: string): WriteResult { + return { documentId, confirmed: true, cost: previewCredits(0), actualCredits: 0 } +} + function isDuplicate(e: unknown): boolean { return e instanceof ConsensusRefusal && e.code === DUPLICATE_UNIQUE_CODE } @@ -100,7 +105,7 @@ export async function writePackManifest( const problem = manifestUrisProblem(input.uris) if (problem) throw new Error(problem) const existing = await findOwnManifest(sdk, repo, auth.identityId, input.packHash) - if (existing !== null) return { documentId: existing, confirmed: true, cost: previewCredits(0), actualCredits: 0 } + if (existing !== null) return alreadyRecorded(existing) const data: Record = { repoId: decodeIdentifier(repo.repoId), packHash: hexToBytes(input.packHash), @@ -120,7 +125,7 @@ export async function writePackManifest( if (!isDuplicate(e)) throw e const id = await findOwnManifest(sdk, repo, auth.identityId, input.packHash) if (id === null) throw e - return { documentId: id, confirmed: true, cost: previewCredits(0), actualCredits: 0 } + return alreadyRecorded(id) } } diff --git a/forge-web/lib/storage/cors.ts b/forge-web/lib/storage/cors.ts index 70be53c7b..e846dd6ba 100644 --- a/forge-web/lib/storage/cors.ts +++ b/forge-web/lib/storage/cors.ts @@ -13,8 +13,9 @@ import type { ProviderId } from './profiles' /** The headers a signed browser request sends that are not CORS-safelisted. */ -export const SIGNED_HEADERS = ['authorization', 'content-type', 'range', 'x-amz-content-sha256', 'x-amz-date', 'x-amz-security-token'] as const +const SIGNED_HEADERS = ['authorization', 'content-type', 'range', 'x-amz-content-sha256', 'x-amz-date', 'x-amz-security-token'] as const const EXPOSE = ['Content-Range', 'Content-Length', 'ETag'] as const +const EXPOSE_LOWER = EXPOSE.map((h) => h.toLowerCase()) /** A fix: where to apply it, and the text to paste or run. */ export interface CorsFix { @@ -48,8 +49,8 @@ export function corsFix(provider: ProviderId, bucket: string, origin: string): C where: `Save as cors.json, then run: b2 bucket update --cors-rules "$(cat cors.json)" ${b} allPublic`, text: JSON.stringify( [ - { corsRuleName: 'dashForgeRead', allowedOrigins: ['*'], allowedOperations: ['s3_get', 's3_head', 'b2_download_file_by_name'], allowedHeaders: ['range'], exposeHeaders: EXPOSE.map((h) => h.toLowerCase()), maxAgeSeconds: 86400 }, - { corsRuleName: 'dashForgeWrite', allowedOrigins: [origin], allowedOperations: ['s3_put', 's3_get', 's3_head', 's3_delete'], allowedHeaders: [...SIGNED_HEADERS], exposeHeaders: EXPOSE.map((h) => h.toLowerCase()), maxAgeSeconds: 86400 }, + { corsRuleName: 'dashForgeRead', allowedOrigins: ['*'], allowedOperations: ['s3_get', 's3_head', 'b2_download_file_by_name'], allowedHeaders: ['range'], exposeHeaders: EXPOSE_LOWER, maxAgeSeconds: 86400 }, + { corsRuleName: 'dashForgeWrite', allowedOrigins: [origin], allowedOperations: ['s3_put', 's3_get', 's3_head', 's3_delete'], allowedHeaders: [...SIGNED_HEADERS], exposeHeaders: EXPOSE_LOWER, maxAgeSeconds: 86400 }, ], null, 2, @@ -72,7 +73,7 @@ export function corsFix(provider: ProviderId, bucket: string, origin: string): C } /** `ipfs config` lines allowing this app to call the RPC API and read the gateway. */ -export function kuboCorsLines(origin: string): string { +function kuboCorsLines(origin: string): string { return [ `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin '["${origin}"]'`, `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Methods '["POST"]'`, diff --git a/forge-web/lib/storage/index.ts b/forge-web/lib/storage/index.ts index cb19546a3..bca241664 100644 --- a/forge-web/lib/storage/index.ts +++ b/forge-web/lib/storage/index.ts @@ -8,13 +8,12 @@ import type { EvoSDK } from '@dashevo/evo-sdk' import type { WriteAuth, WriteResult } from '../sdk' import type { V2RepoRef } from '../repo/contract' -import { writePackManifest } from '../repo/push' -import { storeArtifact, type PlatformQuestion, type StoredArtifact, type UploadEvent } from './upload' -import type { StoragePolicy, StorageProfile } from './profiles' +import { writePackManifest, type PackManifestInput } from '../repo/push' +import { storeArtifact, type StoreOptions, type StoredArtifact } from './upload' export * from './profiles' export * from './store' -export { corsFix, kuboCorsLines, type CorsFix } from './cors' +export { corsFix, type CorsFix } from './cors' export { probeProfile, IPFS_ROWS, S3_ROWS, type ProbeRow, type RowId, type RowState } from './probe' export { PlatformDeclinedError, @@ -23,6 +22,7 @@ export { orderUris, storeArtifact, type PlatformQuestion, + type StoreOptions, type StoredArtifact, type TargetFailure, type UploadEvent, @@ -45,32 +45,11 @@ export async function storeAndRecordPack( auth: WriteAuth, repo: V2RepoRef, bytes: Uint8Array, - meta: { readonly kind: number; readonly objectCount: number; readonly tips?: readonly string[]; readonly supersedes?: readonly string[] }, - opts: { - readonly policy: StoragePolicy | null - readonly profiles: readonly StorageProfile[] - readonly confirmPlatform: (q: PlatformQuestion) => Promise - readonly onStep?: (e: UploadEvent) => void - readonly intent?: string - }, + meta: Pick, + opts: StoreOptions & { readonly intent?: string }, ): Promise { const stored = await storeArtifact(sdk, auth, repo, bytes, opts) - const manifest = await writePackManifest( - sdk, - auth, - repo, - { - packHash: stored.packHash, - kind: meta.kind, - sizeBytes: stored.sizeBytes, - objectCount: meta.objectCount, - chunkCount: stored.chunkCount, - storage: stored.storage, - uris: stored.uris, - ...(meta.tips ? { tips: meta.tips } : {}), - ...(meta.supersedes ? { supersedes: meta.supersedes } : {}), - }, - opts.intent, - ) + const { packHash, sizeBytes, chunkCount, storage, uris } = stored + const manifest = await writePackManifest(sdk, auth, repo, { ...meta, packHash, sizeBytes, chunkCount, storage, uris }, opts.intent) return { stored, manifest } } diff --git a/forge-web/lib/storage/ipfs.ts b/forge-web/lib/storage/ipfs.ts index 7c1c1c2fb..4dc2dbc3c 100644 --- a/forge-web/lib/storage/ipfs.ts +++ b/forge-web/lib/storage/ipfs.ts @@ -50,6 +50,12 @@ async function rpc(s: IpfsSettings, secrets: ProfileSecrets, op: string, pathAnd return text } +/** Let kubo's "not pinned" answer through as '' (nothing pinned); rethrow anything else. */ +function ignoreNotPinned(e: unknown): string { + if (e instanceof IpfsError && /not pinned/.test(e.message)) return '' + throw e +} + /** kubo's version (`/api/v0/version`), a cheap reachability check. */ export async function kuboVersion(s: IpfsSettings, secrets: ProfileSecrets): Promise { const text = await rpc(s, secrets, 'version', 'version') @@ -67,10 +73,7 @@ export async function addVerified(s: IpfsSettings, secrets: ProfileSecrets, byte if (cid !== expected) { throw new IpfsError('ipfs add', `kubo returned CID ${cid || '(none)'} but these bytes derive to ${expected} under the pinned import parameters; refusing to record it`) } - const pinned = await rpc(s, secrets, 'pin check', `pin/ls?arg=${cid}&type=recursive`).catch((e: unknown) => { - if (e instanceof IpfsError && /not pinned/.test(e.message)) return '' - throw e - }) + const pinned = await rpc(s, secrets, 'pin check', `pin/ls?arg=${cid}&type=recursive`).catch(ignoreNotPinned) if (!pinned.includes(cid)) throw new IpfsError('pin check', `kubo added ${cid} but does not report it pinned; a gc would drop it`) return cid } @@ -78,10 +81,7 @@ export async function addVerified(s: IpfsSettings, secrets: ProfileSecrets, byte /** Unpin `cid` (the wizard's probe cleanup). A CID that is not pinned is fine. */ export async function unpin(s: IpfsSettings, secrets: ProfileSecrets, cid: string): Promise { if (!isCid(cid)) throw new IpfsError('unpin', 'not a CID') - await rpc(s, secrets, 'unpin', `pin/rm?arg=${cid}`).catch((e: unknown) => { - if (e instanceof IpfsError && /not pinned/.test(e.message)) return '' - throw e - }) + await rpc(s, secrets, 'unpin', `pin/rm?arg=${cid}`).catch(ignoreNotPinned) } /** The node's announced multiaddrs, handed to a pinning service as `origins`. */ diff --git a/forge-web/lib/storage/probe.ts b/forge-web/lib/storage/probe.ts index 4eeb19de5..8d64135cd 100644 --- a/forge-web/lib/storage/probe.ts +++ b/forge-web/lib/storage/probe.ts @@ -12,11 +12,11 @@ * host is up. */ -import { cidV1RawLeaves } from './cid' import { S3Error, deleteObject, getObject, publicObjectUrl, putObject, type S3Settings } from './s3' import { IpfsError, addVerified, gatewayUrl, kuboVersion, pinningReachable, unpin, type IpfsSettings } from './ipfs' import { normalizedPrefix, type ProfileSecrets, type StorageProfile } from './profiles' import { sha256Hex } from './sigv4' +import { bytesEqual, errText } from './util' export type RowId = 'put' | 'get' | 'public' | 'cors-range' | 'cors-put' | 'delete' | 'api' | 'add' | 'gateway' | 'pinning' | 'unpin' export type RowState = 'pending' | 'running' | 'ok' | 'fail' | 'skipped' @@ -48,7 +48,7 @@ export const IPFS_ROWS: readonly { id: RowId; label: string }[] = [ ] /** Whether `url`'s host answers at all (an opaque no-cors request; never readable). */ -export async function reachable(url: string): Promise { +async function reachable(url: string): Promise { try { await fetch(url, { mode: 'no-cors', credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(10_000) }) return true @@ -63,22 +63,12 @@ function probeBody(): Uint8Array { return new TextEncoder().encode(`dash-forge web storage probe ${Date.now()} ${nonce}\n`) } -function equal(a: Uint8Array, b: Uint8Array): boolean { - return a.length === b.length && a.every((x, i) => x === b[i]) -} - type Report = (id: RowId, state: RowState, detail: string, cors?: boolean) => void -/** Run the S3 test, reporting each row as it settles. Resolves with whether every row passed. */ -export async function probeS3(s: S3Settings, secrets: ProfileSecrets, report: Report): Promise { +/** Run the S3 test, reporting each row as it settles. */ +async function probeS3(s: S3Settings, secrets: ProfileSecrets, report: Report): Promise { const body = probeBody() const key = `${normalizedPrefix(s.prefix)}probe/forge-web-test-${await sha256Hex(body)}.txt` - let ok = true - const fail = (id: RowId, detail: string, cors = false): void => { - ok = false - report(id, 'fail', detail, cors) - } - const skipRest = (from: RowId[], why: string): void => from.forEach((id) => report(id, 'skipped', why)) // 1 + 5: a PUT that lands proves the PUT preflight passed; one refused before any answer is // CORS when the endpoint is reachable. @@ -90,40 +80,41 @@ export async function probeS3(s: S3Settings, secrets: ProfileSecrets, report: Re } catch (e) { const blocked = e instanceof S3Error && e.status === 0 && (await reachable(s.endpoint)) if (blocked) { - fail('put', 'the browser refused to send it (see the CORS row)') - fail('cors-put', 'the bucket’s CORS rules do not allow a signed PUT from this origin', true) + report('put', 'fail', 'the browser refused to send it (see the CORS row)') + report('cors-put', 'fail', 'the bucket’s CORS rules do not allow a signed PUT from this origin', true) } else { - fail('put', errText(e)) + report('put', 'fail', errText(e)) report('cors-put', 'skipped', 'needs a PUT that reaches the bucket') } - skipRest(['get', 'public', 'cors-range', 'delete'], 'needs the probe object') - return false + for (const id of ['get', 'public', 'cors-range', 'delete'] as const) report(id, 'skipped', 'needs the probe object') + return } report('get', 'running', '') try { const got = await getObject(s, secrets, key) - if (equal(got, body)) report('get', 'ok', 'read back byte for byte') - else fail('get', 'the bucket returned different bytes (a cache or proxy in front of it?)') + if (bytesEqual(got, body)) report('get', 'ok', 'read back byte for byte') + else report('get', 'fail', 'the bucket returned different bytes (a cache or proxy in front of it?)') } catch (e) { - fail('get', errText(e), e instanceof S3Error && e.status === 0) + report('get', 'fail', errText(e), e instanceof S3Error && e.status === 0) } const publicUrl = publicObjectUrl(s, key) report('public', 'running', '') + const publicHost = new URL(publicUrl).host let publicOk = false try { const resp = await fetch(publicUrl, { credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(20_000) }) const got = new Uint8Array(await resp.arrayBuffer()) - if (!resp.ok) fail('public', `HTTP ${resp.status} from ${new URL(publicUrl).host}: the objects are not publicly readable`) - else if (!equal(got, body)) fail('public', `${new URL(publicUrl).host} served different bytes (check the public URL)`) + if (!resp.ok) report('public', 'fail', `HTTP ${resp.status} from ${publicHost}: the objects are not publicly readable`) + else if (!bytesEqual(got, body)) report('public', 'fail', `${publicHost} served different bytes (check the public URL)`) else { publicOk = true - report('public', 'ok', `anonymous GET ${new URL(publicUrl).host}`) + report('public', 'ok', `anonymous GET ${publicHost}`) } } catch { const up = await reachable(publicUrl) - fail('public', up ? 'the browser was not allowed to read it: no Access-Control-Allow-Origin on GET' : `${new URL(publicUrl).host} did not answer`, up) + report('public', 'fail', up ? 'the browser was not allowed to read it: no Access-Control-Allow-Origin on GET' : `${publicHost} did not answer`, up) } report('cors-range', 'running', '') @@ -132,11 +123,11 @@ export async function probeS3(s: S3Settings, secrets: ProfileSecrets, report: Re try { const resp = await fetch(publicUrl, { headers: { Range: 'bytes=0-9' }, credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(20_000) }) const range = resp.headers.get('content-range') - if (resp.status !== 206) fail('cors-range', `a ranged GET returned ${resp.status} instead of 206: browsing needs HTTP Range support on the public URL`) - else if (range === null) fail('cors-range', 'Content-Range is not exposed to scripts (Access-Control-Expose-Headers)', true) + if (resp.status !== 206) report('cors-range', 'fail', `a ranged GET returned ${resp.status} instead of 206: browsing needs HTTP Range support on the public URL`) + else if (range === null) report('cors-range', 'fail', 'Content-Range is not exposed to scripts (Access-Control-Expose-Headers)', true) else report('cors-range', 'ok', 'Range allowed, Content-Range exposed') } catch { - fail('cors-range', 'the CORS preflight for a Range request was refused: allow the Range request header', true) + report('cors-range', 'fail', 'the CORS preflight for a Range request was refused: allow the Range request header', true) } } @@ -145,26 +136,20 @@ export async function probeS3(s: S3Settings, secrets: ProfileSecrets, report: Re await deleteObject(s, secrets, key) report('delete', 'ok', 'probe removed') } catch (e) { - fail('delete', `${errText(e)}. The probe object (a few bytes) stays at ${key}; delete it by hand.`, e instanceof S3Error && e.status === 0) + report('delete', 'fail', `${errText(e)}. The probe object (a few bytes) stays at ${key}; delete it by hand.`, e instanceof S3Error && e.status === 0) } - return ok } /** Run the IPFS test (kubo, plus the pinning service for a pinning profile). */ -export async function probeIpfs(s: IpfsSettings, secrets: ProfileSecrets, report: Report): Promise { - let ok = true - const fail = (id: RowId, detail: string, cors = false): void => { - ok = false - report(id, 'fail', detail, cors) - } +async function probeIpfs(s: IpfsSettings, secrets: ProfileSecrets, report: Report): Promise { report('api', 'running', '') try { report('api', 'ok', `kubo ${await kuboVersion(s, secrets)}`) } catch (e) { const up = await reachable(s.api) - fail('api', up ? `${errText(e)}` : `${new URL(s.api).host} did not answer`, up) + report('api', 'fail', up ? errText(e) : `${new URL(s.api).host} did not answer`, up) for (const id of ['add', 'gateway', 'pinning', 'unpin'] as const) report(id, 'skipped', 'needs the kubo API') - return false + return } const body = probeBody() @@ -174,7 +159,7 @@ export async function probeIpfs(s: IpfsSettings, secrets: ProfileSecrets, report cid = await addVerified(s, secrets, body) report('add', 'ok', `${cid} matches the local derivation and is pinned`) } catch (e) { - fail('add', errText(e)) + report('add', 'fail', errText(e)) } report('gateway', 'running', '') @@ -184,12 +169,12 @@ export async function probeIpfs(s: IpfsSettings, secrets: ProfileSecrets, report try { const resp = await fetch(url, { headers: { Range: `bytes=0-${body.length - 1}` }, credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(20_000) }) const got = new Uint8Array(await resp.arrayBuffer()) - if (!resp.ok) fail('gateway', `HTTP ${resp.status} from the gateway`) - else if (!equal(got, body)) fail('gateway', 'the gateway served different bytes') + if (!resp.ok) report('gateway', 'fail', `HTTP ${resp.status} from the gateway`) + else if (!bytesEqual(got, body)) report('gateway', 'fail', 'the gateway served different bytes') else report('gateway', 'ok', `re-read through ${new URL(url).host}`) } catch { const up = await reachable(url) - fail('gateway', up ? 'the gateway refused this origin (CORS)' : 'the gateway did not answer', up) + report('gateway', 'fail', up ? 'the gateway refused this origin (CORS)' : 'the gateway did not answer', up) } } @@ -200,7 +185,7 @@ export async function probeIpfs(s: IpfsSettings, secrets: ProfileSecrets, report await pinningReachable(s, secrets) report('pinning', 'ok', 'the service accepted the token') } catch (e) { - fail('pinning', errText(e), e instanceof IpfsError && /CORS/.test(e.message)) + report('pinning', 'fail', errText(e), e instanceof IpfsError && /CORS/.test(e.message)) } } @@ -211,23 +196,24 @@ export async function probeIpfs(s: IpfsSettings, secrets: ProfileSecrets, report await unpin(s, secrets, cid) report('unpin', 'ok', 'probe unpinned (a gc removes it)') } catch (e) { - fail('unpin', errText(e)) + report('unpin', 'fail', errText(e)) } } - return ok } -/** Run the test for any profile kind. A Platform profile needs none. */ +/** + * Run the test for any profile kind, reporting each row as it settles. Resolves with whether + * every row passed (no row failed). A Platform profile needs none. + */ export async function probeProfile(p: StorageProfile, report: Report): Promise { const s = p.settings if (s.kind === 'platform') return true - if (s.kind === 's3') return probeS3(s, p.secrets, report) - return probeIpfs(s, p.secrets, report) -} - -function errText(e: unknown): string { - return e instanceof Error ? e.message : String(e) + let ok = true + const tracked: Report = (id, state, detail, cors) => { + if (state === 'fail') ok = false + report(id, state, detail, cors) + } + if (s.kind === 's3') await probeS3(s, p.secrets, tracked) + else await probeIpfs(s, p.secrets, tracked) + return ok } - -/** The CID a probe body would get (exposed for tests). */ -export const probeCid = cidV1RawLeaves diff --git a/forge-web/lib/storage/s3.ts b/forge-web/lib/storage/s3.ts index 9fc26a178..e33e03849 100644 --- a/forge-web/lib/storage/s3.ts +++ b/forge-web/lib/storage/s3.ts @@ -115,26 +115,33 @@ async function failed(op: string, resp: Response): Promise { } catch { /* body unreadable */ } - const hint = - resp.status === 403 - ? ' — check the key, its bucket permissions, the region (R2 wants auto) and the clock' - : resp.status === 404 - ? ' — check the bucket name and endpoint' - : resp.status === 301 || resp.status === 307 - ? ' — the bucket lives in another region; set that endpoint' - : '' - return new S3Error(op, resp.status, `${op} failed: HTTP ${resp.status}${code ? ` ${code}` : ''}${hint}`) + return new S3Error(op, resp.status, `${op} failed: HTTP ${resp.status}${code ? ` ${code}` : ''}${statusHint(resp.status)}`) +} + +/** What an S3 error status usually means, as a suffix for the message ('' when nothing useful). */ +function statusHint(status: number): string { + switch (status) { + case 403: + return ' — check the key, its bucket permissions, the region (R2 wants auto) and the clock' + case 404: + return ' — check the bucket name and endpoint' + case 301: + case 307: + return ' — the bucket lives in another region; set that endpoint' + default: + return '' + } } /** Signed PUT of `bytes` at `key`. */ export async function putObject(s: S3Settings, secrets: ProfileSecrets, key: string, bytes: Uint8Array, contentType = 'application/octet-stream', signal?: AbortSignal): Promise { - const resp = await send(s, secrets, 'signed PUT', 'PUT', key, { body: bytes, contentType, ...(signal ? { signal } : {}) }) + const resp = await send(s, secrets, 'signed PUT', 'PUT', key, { body: bytes, contentType, signal }) if (!resp.ok) throw await failed('signed PUT', resp) } /** Signed GET of `key` (whole object, or a `Range`). */ export async function getObject(s: S3Settings, secrets: ProfileSecrets, key: string, range?: string, signal?: AbortSignal): Promise { - const resp = await send(s, secrets, 'signed GET', 'GET', key, { ...(range ? { range } : {}), ...(signal ? { signal } : {}) }) + const resp = await send(s, secrets, 'signed GET', 'GET', key, { range, signal }) if (!resp.ok) throw await failed('signed GET', resp) return new Uint8Array(await resp.arrayBuffer()) } diff --git a/forge-web/lib/storage/upload.ts b/forge-web/lib/storage/upload.ts index dbcbb718a..b1fbfb043 100644 --- a/forge-web/lib/storage/upload.ts +++ b/forge-web/lib/storage/upload.ts @@ -19,11 +19,12 @@ import type { EvoSDK } from '@dashevo/evo-sdk' import type { WriteAuth } from '../sdk' import { estimateChunkCredits } from '../sdk/cost' import type { V2RepoRef } from '../repo/contract' -import { putPlatformChunks } from '../repo/push' +import { manifestUrisProblem, putPlatformChunks } from '../repo/push' import { addVerified, gatewayUrl, remotePin } from './ipfs' import { artifactKey, type StoragePolicy, type StorageProfile } from './profiles' import { getObject, headObject, publicObjectUrl, putObject, s3Uri, type S3Settings } from './s3' import { sha256Hex } from './sigv4' +import { bytesEqual, errText } from './util' /** Artifacts up to this size are verified by a full re-download and SHA-256. */ export const FULL_VERIFY_MAX = 16 * 1024 * 1024 @@ -88,10 +89,6 @@ export class PlatformDeclinedError extends Error { } } -function equal(a: Uint8Array, b: Uint8Array): boolean { - return a.length === b.length && a.every((x, i) => x === b[i]) -} - /** Re-read an S3 copy and check it is exactly `bytes`. */ async function verifyS3(s: S3Settings, p: StorageProfile, key: string, bytes: Uint8Array, hashHex: string): Promise { if (bytes.length <= FULL_VERIFY_MAX) { @@ -106,7 +103,7 @@ async function verifyS3(s: S3Settings, p: StorageProfile, key: string, bytes: Ui [Math.max(0, bytes.length - EDGE_WINDOW), bytes.length], ] as const) { const got = await getObject(s, p.secrets, key, `bytes=${start}-${end - 1}`) - if (!equal(got, bytes.subarray(start, end))) throw new Error(`re-read of ${key} bytes ${start}..${end} differ from the artifact`) + if (!bytesEqual(got, bytes.subarray(start, end))) throw new Error(`re-read of ${key} bytes ${start}..${end} differ from the artifact`) } } @@ -126,7 +123,7 @@ async function storeExternal(p: StorageProfile, bytes: Uint8Array, hashHex: stri try { await verifyS3(s, p, key, bytes, hashHex) } catch (second) { - throw new Error(`${msg(second)} (also after a re-upload; first attempt: ${msg(first)})`) + throw new Error(`${errText(second)} (also after a re-upload; first attempt: ${errText(first)})`) } } return [publicObjectUrl(s, key), s3Uri(s, key)] @@ -146,10 +143,6 @@ async function storeExternal(p: StorageProfile, bytes: Uint8Array, hashHex: stri throw new Error('not an external profile') } -function msg(e: unknown): string { - return e instanceof Error ? e.message : String(e) -} - /** * Every recorded URI, de-duplicated: the `platform://` locator first (released helpers read * `uris[0]` of a storage-0 manifest as the chunk locator), then public http(s) URLs, then the @@ -172,13 +165,21 @@ export function orderUris(groups: readonly (readonly string[])[]): string[] { * (readers without that profile cannot use them); then an error, never a silent truncation. */ export function fitManifestUris(uris: readonly string[]): string[] { - const fits = (list: readonly string[]): boolean => list.length > 0 && list.length <= 8 && list.every((u) => new TextEncoder().encode(u).length <= 300) + const fits = (list: readonly string[]): boolean => manifestUrisProblem(list) === null if (fits(uris)) return [...uris] const noS3 = uris.filter((u) => !u.startsWith('s3://')) if (fits(noS3)) return noS3 throw new Error("the confirmed copies' URIs do not fit a manifest (at most 8, each up to 300 bytes); use shorter public URLs or fewer targets") } +/** What {@link storeArtifact} needs besides the bytes. */ +export interface StoreOptions { + readonly policy: StoragePolicy | null + readonly profiles: readonly StorageProfile[] + readonly confirmPlatform: (q: PlatformQuestion) => Promise + readonly onStep?: (e: UploadEvent) => void +} + /** * Store `bytes` under `policy`, verifying every copy, and return what the manifest records. * With no policy, or when the policy is not met and it allows the Platform fallback, the user @@ -190,12 +191,7 @@ export async function storeArtifact( auth: WriteAuth, repo: V2RepoRef, bytes: Uint8Array, - opts: { - readonly policy: StoragePolicy | null - readonly profiles: readonly StorageProfile[] - readonly confirmPlatform: (q: PlatformQuestion) => Promise - readonly onStep?: (e: UploadEvent) => void - }, + opts: StoreOptions, ): Promise { const hashHex = await sha256Hex(bytes) const step = opts.onStep ?? (() => undefined) @@ -203,6 +199,13 @@ export async function storeArtifact( const failures: TargetFailure[] = [] let chunkCount = 0 + const fail = (target: string, e: unknown): void => { + failures.push({ target, reason: errText(e) }) + step({ target, phase: 'failed', reason: errText(e) }) + } + const askPlatform = (reason: string): Promise => + opts.confirmPlatform({ bytes: bytes.length, estimateCredits: estimateChunkCredits(bytes.length), reason }) + const platformCopy = async (target: string): Promise => { step({ target, phase: 'start' }) try { @@ -211,16 +214,14 @@ export async function storeArtifact( confirmed.push({ target, uris: [r.locator], platform: true }) step({ target, phase: 'done', uris: [r.locator] }) } catch (e) { - failures.push({ target, reason: msg(e) }) - step({ target, phase: 'failed', reason: msg(e) }) + fail(target, e) throw e } } const policy = opts.policy if (policy === null || policy.targets.length === 0) { - const ok = await opts.confirmPlatform({ bytes: bytes.length, estimateCredits: estimateChunkCredits(bytes.length), reason: 'No storage is configured for browser pushes to this repo.' }) - if (!ok) throw new PlatformDeclinedError() + if (!(await askPlatform('No storage is configured for browser pushes to this repo.'))) throw new PlatformDeclinedError() await platformCopy('platform') } else { const byName = new Map(opts.profiles.map((p) => [p.name, p])) @@ -236,26 +237,20 @@ export async function storeArtifact( confirmed.push({ target: name, uris, platform: false }) step({ target: name, phase: 'done', uris }) } catch (e) { - failures.push({ target: name, reason: msg(e) }) - step({ target: name, phase: 'failed', reason: msg(e) }) + fail(name, e) } }), ) // Platform targets only if they can still make up the policy: never pay for chunks // whose push is about to fail anyway. - for (let i = 0; i < onChain.length; i++) { + for (const [i, { name }] of onChain.entries()) { if (confirmed.length + (onChain.length - i) < policy.replicas) break - const t = onChain[i] as { name: string } - await platformCopy(t.name).catch(() => undefined) + await platformCopy(name).catch(() => undefined) } if (confirmed.length < policy.replicas) { - if (!policy.platformFallback || confirmed.some((c) => c.platform)) throw new ReplicationError(policy.replicas, confirmed.map((c) => c.target), failures) - const ok = await opts.confirmPlatform({ - bytes: bytes.length, - estimateCredits: estimateChunkCredits(bytes.length), - reason: `Your storage did not confirm (${failures.map((f) => f.target).join(', ')}).`, - }) - if (!ok) throw new ReplicationError(policy.replicas, confirmed.map((c) => c.target), failures) + const notMet = (): ReplicationError => new ReplicationError(policy.replicas, confirmed.map((c) => c.target), failures) + if (!policy.platformFallback || confirmed.some((c) => c.platform)) throw notMet() + if (!(await askPlatform(`Your storage did not confirm (${failures.map((f) => f.target).join(', ')}).`))) throw notMet() await platformCopy('platform (fallback)') } } diff --git a/forge-web/lib/storage/util.ts b/forge-web/lib/storage/util.ts new file mode 100644 index 000000000..b257a2003 --- /dev/null +++ b/forge-web/lib/storage/util.ts @@ -0,0 +1,11 @@ +/** Small helpers shared by the live test (`./probe`) and the upload path (`./upload`). */ + +/** Whether `a` and `b` hold the same bytes. */ +export function bytesEqual(a: Uint8Array, b: Uint8Array): boolean { + return a.length === b.length && a.every((x, i) => x === b[i]) +} + +/** The message of a thrown value, as a row detail or a failure reason. */ +export function errText(e: unknown): string { + return e instanceof Error ? e.message : String(e) +} From f1a5da4d19d9820056451140dd2b942b37fd9c9d Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 26 Sep 2026 17:19:31 -0500 Subject: [PATCH 3/5] fix(web): address the storage security review Only public https addresses are recorded on chain or fetched by readers (lib/net.ts): profiles refuse a loopback or private public URL for uploads, fitManifestUris refuses non-https URIs, externalFetchUrls skips them, and the CSP opens loopback http only in devnet and dev builds. A copy counts only after an anonymous head-and-tail read through its public URL. Every Platform write asks first with its price, and the policy error says whether chunks were written. Requests use a 15 s connect and 120 s idle deadline instead of a 60 s total cap. The kubo fix creates a token limited to add, pin and id and merges this origin into the allowed list; the form explains the admin-API trust. Test results are tied to the exact values tested. Also: the reserved platform name, renames kept in policies, the record and settings written in one transaction with a discard option, secrets never put back into the page, header-unsafe secrets refused without echoing them, redirect vs CORS vs down told apart, live regions. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/app/layout.tsx | 11 +- forge-web/components/storage/profile-form.tsx | 77 ++++-- .../storage/repo-storage-policy.tsx | 15 +- forge-web/components/storage/storage-test.tsx | 48 +++- .../components/storage/storage-wizard.tsx | 106 ++++++-- forge-web/contexts/auth-context.tsx | 6 + forge-web/e2e/storage-wizard.spec.ts | 16 +- forge-web/hooks/use-storage-config.ts | 11 +- forge-web/lib/auth/controller.ts | 9 +- forge-web/lib/auth/vault.ts | 39 ++- forge-web/lib/idb.ts | 27 ++ forge-web/lib/net.ts | 52 ++++ forge-web/lib/storage/cors.ts | 26 +- forge-web/lib/storage/ipfs.ts | 52 +++- forge-web/lib/storage/probe.ts | 167 ++++++++---- forge-web/lib/storage/profiles.ts | 130 +++++++-- forge-web/lib/storage/s3.ts | 82 +++--- forge-web/lib/storage/storage.test.ts | 250 ++++++++++++++---- forge-web/lib/storage/store.ts | 33 ++- forge-web/lib/storage/upload.live.test.ts | 19 +- forge-web/lib/storage/upload.ts | 135 +++++++--- forge-web/lib/storage/util.ts | 100 +++++++ forge-web/lib/view/browse-fallback.test.ts | 20 +- forge-web/lib/view/browse-source.ts | 7 +- 24 files changed, 1118 insertions(+), 320 deletions(-) create mode 100644 forge-web/lib/net.ts diff --git a/forge-web/app/layout.tsx b/forge-web/app/layout.tsx index bc106c2a4..ed9a4fc9c 100644 --- a/forge-web/app/layout.tsx +++ b/forge-web/app/layout.tsx @@ -1,6 +1,7 @@ import type { Metadata } from 'next' import type { ReactNode } from 'react' import { Providers } from '@/components/providers' +import { LOCAL_HTTP_ALLOWED } from '@/lib/net' import './globals.css' export const metadata: Metadata = { @@ -15,9 +16,11 @@ export const metadata: Metadata = { // inline bootstrap scripts. // - frame-ancestors is ignored in a CSP; the host must send it as a header (GitHub // Pages cannot; see docs/guides/identity-and-keys.md). -// - connect-src https:/wss:: DAPI endpoints + IPFS/S3/HTTPS pack backends. Plain http only to -// this machine (a local kubo node or MinIO, the storage guide's defaults); storage profiles -// refuse http anywhere else (`lib/storage/profiles.ts`). +// - connect-src https:/wss:: DAPI endpoints + IPFS/S3/HTTPS pack backends. Plain http to this +// machine is allowed ONLY in devnet and dev builds (`LOCAL_HTTP_ALLOWED`: a local MinIO or +// kubo while developing); production builds never open it, for any page. Readers never +// fetch loopback or private hosts anyway (`lib/net.ts`), and nothing loopback is ever +// recorded on chain. // - worker-src blob:: materialization / search / pack workers run off-main-thread. const CSP = [ "default-src 'self'", @@ -25,7 +28,7 @@ const CSP = [ "style-src 'self' 'unsafe-inline'", "img-src 'self' data: https: blob:", "font-src 'self'", - "connect-src 'self' https: wss: http://127.0.0.1:* http://localhost:*", + `connect-src 'self' https: wss:${LOCAL_HTTP_ALLOWED ? ' http://127.0.0.1:* http://localhost:*' : ''}`, "worker-src 'self' blob:", "child-src 'self' blob:", "frame-ancestors 'none'", diff --git a/forge-web/components/storage/profile-form.tsx b/forge-web/components/storage/profile-form.tsx index 46009ee8d..57aed7f78 100644 --- a/forge-web/components/storage/profile-form.tsx +++ b/forge-web/components/storage/profile-form.tsx @@ -4,10 +4,15 @@ * The per-provider form of the storage wizard (`ux-dx-spec.md` §3.1 step 2): the fields the * guide names, each with a "where to find this" hint. Secret fields carry a lock and say where * they go. Values live in component state only until saved into the vault. + * + * Editing a saved profile never puts its stored secrets back into the page: secret fields start + * empty, and an empty field keeps the stored value. Autocomplete is off everywhere, so a + * password manager neither offers to generate a value nor saves a bucket secret to a synced + * store. */ import { useId, useState } from 'react' -import { ExternalLink, Lock } from 'lucide-react' +import { AlertTriangle, ExternalLink, Lock } from 'lucide-react' import { profileProblem, providerPreset, @@ -19,39 +24,48 @@ import { import { Input } from '@/components/ui/input' const SECRET_NOTE = 'Stored encrypted in this browser only. Never sent to Forge (there is no Forge server) and never written on-chain.' +const KEEP_NOTE = 'Leave empty to keep the stored value.' type Values = Record +const SECRET_KEYS: readonly (keyof ProfileSecrets)[] = ['accessKeyId', 'secretAccessKey', 'sessionToken', 'apiAuth', 'pinningToken'] /** Providers whose suggested profile name is the provider id itself (the rest get `-main`). */ const BARE_NAMES: readonly ProviderId[] = ['aws', 'minio', 'kubo', 'pinning'] function initialValues(provider: ProviderId, existing: StorageProfile | null): Values { const preset = providerPreset(provider) + const empty: Values = + preset.kind === 's3' + ? { endpoint: '', region: 'us-east-1', bucket: '', pathStyle: true, publicUrl: '', prefix: '', accessKeyId: '', secretAccessKey: '', sessionToken: '' } + : { api: '', gateway: '', publicGateway: '', pinningEndpoint: '', apiAuth: '', pinningToken: '' } if (existing) { - const out: Values = { name: existing.name } - for (const [k, v] of [...Object.entries(existing.settings), ...Object.entries(existing.secrets)]) { + // Public settings only; secrets stay out of the DOM. + const out: Values = { ...empty, name: existing.name } + for (const [k, v] of Object.entries(existing.settings)) { if (typeof v === 'string' || typeof v === 'boolean') out[k] = v } return out } - const base: Values = { name: BARE_NAMES.includes(provider) ? provider : `${provider}-main` } - const empty: Values = - preset.kind === 's3' - ? { endpoint: '', region: 'us-east-1', bucket: '', pathStyle: true, publicUrl: '', prefix: '', accessKeyId: '', secretAccessKey: '', sessionToken: '' } - : { api: '', gateway: '', publicGateway: '', pinningEndpoint: '', apiAuth: '', pinningToken: '' } const defaults: Values = {} for (const [k, v] of Object.entries(preset.defaults)) if (v !== undefined) defaults[k] = v - return { ...base, ...empty, ...defaults } + return { name: BARE_NAMES.includes(provider) ? provider : `${provider}-main`, ...empty, ...defaults } } const str = (v: string | boolean | undefined): string => (typeof v === 'string' ? v.trim() : '') /** A URL field, trimmed and without trailing slashes. */ const url = (v: string | boolean | undefined): string => str(v).replace(/\/+$/, '') -/** The profile the form's values describe. */ -export function profileFromValues(provider: ProviderId, v: Values): StorageProfile { +/** + * The profile the form's values describe. A secret field left empty while editing keeps the + * stored value (`kept`). + */ +export function profileFromValues(provider: ProviderId, v: Values, kept: ProfileSecrets = {}): StorageProfile { const preset = providerPreset(provider) - const secret = (k: keyof ProfileSecrets): Partial => (str(v[k]) ? { [k]: str(v[k]) } : {}) + const secrets: Record = {} + for (const k of SECRET_KEYS) { + const value = str(v[k]) || kept[k] + if (value) secrets[k] = value + } let settings: ProfilePublic if (preset.kind === 's3') { settings = { @@ -64,18 +78,25 @@ export function profileFromValues(provider: ProviderId, v: Values): StorageProfi publicUrl: url(v['publicUrl']), prefix: str(v['prefix']), } - return { name: str(v['name']), settings, secrets: { ...secret('accessKeyId'), ...secret('secretAccessKey'), ...secret('sessionToken') } } + const { accessKeyId, secretAccessKey, sessionToken } = secrets + return { name: str(v['name']), settings, secrets: { ...(accessKeyId ? { accessKeyId } : {}), ...(secretAccessKey ? { secretAccessKey } : {}), ...(sessionToken ? { sessionToken } : {}) } } } - if (preset.kind === 'platform') return { name: str(v['name']) || 'platform', settings: { kind: 'platform', provider: 'platform' }, secrets: {} } + if (preset.kind === 'platform') return { name: 'platform', settings: { kind: 'platform', provider: 'platform' }, secrets: {} } settings = { kind: preset.kind === 'ipfs-pinning-service' ? 'ipfs-pinning-service' : 'ipfs-kubo', provider: provider as 'kubo' | 'pinning', api: url(v['api']), gateway: url(v['gateway']), publicGateway: url(v['publicGateway']), - pinningEndpoint: url(v['pinningEndpoint']), + pinningEndpoint: preset.kind === 'ipfs-pinning-service' ? url(v['pinningEndpoint']) : '', } - return { name: str(v['name']), settings, secrets: { ...secret('apiAuth'), ...secret('pinningToken') } } + const { apiAuth, pinningToken } = secrets + return { name: str(v['name']), settings, secrets: { ...(apiAuth ? { apiAuth } : {}), ...(pinningToken && preset.kind === 'ipfs-pinning-service' ? { pinningToken } : {}) } } +} + +/** The draft a form starts from (so an edit can be tested before any field changes). */ +export function initialDraft(provider: ProviderId, existing: StorageProfile | null): StorageProfile { + return profileFromValues(provider, initialValues(provider, existing), existing?.secrets ?? {}) } interface FieldSpec { @@ -107,7 +128,7 @@ function fieldsFor(provider: ProviderId): FieldSpec[] { { key: 'api', label: 'kubo RPC API' }, { key: 'gateway', label: 'Gateway (verification re-read)', optional: true }, { key: 'publicGateway', label: 'Public gateway (recorded for browsers)', optional: true }, - { key: 'apiAuth', label: 'API Authorization header', secret: true, optional: true, placeholder: 'Basic …' }, + { key: 'apiAuth', label: 'API Authorization header', secret: true, optional: true, placeholder: 'Bearer …' }, ] if (kind === 'ipfs-pinning-service') { f.push({ key: 'pinningEndpoint', label: 'Pinning Service API URL' }, { key: 'pinningToken', label: 'Pinning access token', secret: true }) @@ -128,18 +149,28 @@ export function ProfileForm({ const preset = providerPreset(provider) const [values, setValues] = useState(() => initialValues(provider, existing)) const idBase = useId() + const kept = existing?.secrets ?? {} const set = (k: string, v: string | boolean): void => { const next = { ...values, [k]: v } setValues(next) - const p = profileFromValues(provider, next) + const p = profileFromValues(provider, next, kept) onChange(p, profileProblem(p)) } return ( -
+
+ {preset.kind === 'ipfs-kubo' || preset.kind === 'ipfs-pinning-service' ? ( +
+ + + kubo’s RPC API is the node’s admin interface. Letting this web app call it means trusting this site with your node, unless kubo limits the token it uses. Create a token restricted to add, pin-check, unpin and id (kubo API.Authorizations with AllowedPaths; the test prints the commands if the node refuses this origin) and paste it as the API Authorization header. + +
+ ) : null} {fieldsFor(provider).map((f) => { const hint = preset.hints[f.key] const id = `${idBase}-${f.key}` + const keepable = f.secret === true && existing !== null && kept[f.key as keyof ProfileSecrets] !== undefined return (
set(f.key, e.target.value)} aria-describedby={hint || f.secret ? `${id}-hint` : undefined} @@ -161,6 +195,7 @@ export function ProfileForm({ {hint || f.secret ? (

{f.secret ? SECRET_NOTE : null} + {keepable ? ` ${KEEP_NOTE}` : null} {f.secret && hint ? ' ' : null} {hint ? ( <> diff --git a/forge-web/components/storage/repo-storage-policy.tsx b/forge-web/components/storage/repo-storage-policy.tsx index 02504f4b0..f172392fe 100644 --- a/forge-web/components/storage/repo-storage-policy.tsx +++ b/forge-web/components/storage/repo-storage-policy.tsx @@ -107,17 +107,20 @@ export function RepoStoragePolicy({ repoId }: { repoId: string }): JSX.Element { variant="ghost" disabled={!storable} onClick={async () => { - await save(withRepoPolicy(config, repoId, null)) - setTargets(null) - setChoice(null) - setMsg('Back to your default.') + try { + await save(withRepoPolicy(config, repoId, null)) + setTargets(null) + setChoice(null) + setMsg('Back to your default.') + } catch (e) { + setMsg(e instanceof Error ? e.message : String(e)) + } }} > Use my default ) : null} - {problem && shown.length > 0 ? {problem} : null} - {msg ? {msg} : null} + {msg ?? (problem && shown.length > 0 ? problem : '')}

) diff --git a/forge-web/components/storage/storage-test.tsx b/forge-web/components/storage/storage-test.tsx index 6c1fa9d8a..1ec03f9ef 100644 --- a/forge-web/components/storage/storage-test.tsx +++ b/forge-web/components/storage/storage-test.tsx @@ -6,7 +6,7 @@ * the provider, prefilled with the bucket and this app's origin, with a Re-test button. */ -import { useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { AlertTriangle, CheckCircle2, CircleDashed, Loader2, MinusCircle, RotateCw, XCircle } from 'lucide-react' import { IPFS_ROWS, @@ -36,26 +36,51 @@ function rowsFor(p: StorageProfile): ProbeRow[] { return spec.map((r) => ({ ...r, state: 'pending', detail: '' })) } -/** Run and show the live test for `profile`; `onDone` gets whether every row passed. */ -export function StorageTest({ profile, onDone }: { profile: StorageProfile; onDone: (ok: boolean) => void }): JSX.Element { +/** A stable fingerprint of what a test ran against (settings and secrets). */ +export function profileSnapshot(p: StorageProfile): string { + return JSON.stringify([p.name, p.settings, p.secrets]) +} + +/** + * Run and show the live test for `profile`. `onDone(ok, snapshot)` reports whether every row + * passed, with the {@link profileSnapshot} the run tested, so the caller credits the result only + * to those exact values. The parent keys this component by the snapshot, so an edit remounts it + * (no stale green rows); a run still in flight when that happens reports into the unmounted + * copy and is dropped. + */ +export function StorageTest({ profile, onDone }: { profile: StorageProfile; onDone: (ok: boolean, snapshot: string) => void }): JSX.Element { const [rows, setRows] = useState(null) const [running, setRunning] = useState(false) + const [summary, setSummary] = useState('') + const mounted = useRef(true) + useEffect(() => { + mounted.current = true + return () => { + mounted.current = false + } + }, []) const origin = typeof window === 'undefined' ? 'https://forge.dashhq.org' : window.location.origin const run = async (): Promise => { if (running) return + const snapshot = profileSnapshot(profile) setRunning(true) + setSummary('Testing…') setRows(rowsFor(profile)) - const report = (id: RowId, state: RowState, detail: string, cors?: boolean): void => + const report = (id: RowId, state: RowState, detail: string, cors?: boolean): void => { + if (!mounted.current) return setRows((prev) => (prev ?? []).map((r) => (r.id === id ? { ...r, state, detail, ...(cors ? { cors } : {}) } : r))) + } + let ok = false try { - onDone(await probeProfile(profile, report)) + ok = await probeProfile(profile, report) } catch (e) { - onDone(false) - setRows((prev) => (prev ?? []).map((r) => (r.state === 'running' || r.state === 'pending' ? { ...r, state: 'fail', detail: errText(e) } : r))) - } finally { - setRunning(false) + if (mounted.current) setRows((prev) => (prev ?? []).map((r) => (r.state === 'running' || r.state === 'pending' ? { ...r, state: 'fail', detail: errText(e) } : r))) } + if (!mounted.current) return + setRunning(false) + setSummary(ok ? 'Test finished: every check passed.' : 'Test finished: some checks failed.') + onDone(ok, snapshot) } const corsFailed = rows?.some((r) => r.state === 'fail' && r.cors) ?? false @@ -72,7 +97,7 @@ export function StorageTest({ profile, onDone }: { profile: StorageProfile; onDo
{rows ? ( -
    +
      {rows.map((r) => { const m = META[r.state] return ( @@ -86,6 +111,9 @@ export function StorageTest({ profile, onDone }: { profile: StorageProfile; onDo })}
    ) : null} +

    + {summary} +

    {corsFailed && fix.text ? (

    diff --git a/forge-web/components/storage/storage-wizard.tsx b/forge-web/components/storage/storage-wizard.tsx index 35aa9c6ad..1cea62938 100644 --- a/forge-web/components/storage/storage-wizard.tsx +++ b/forge-web/components/storage/storage-wizard.tsx @@ -8,7 +8,7 @@ * Everything saved is sealed in the vault with this browser's key (`lib/storage/store.ts`). */ -import { useState } from 'react' +import { useId, useState } from 'react' import { Cloud, Database, HardDrive, Link2, Pencil, Server, Trash2, Waypoints } from 'lucide-react' import { PROVIDERS, @@ -16,7 +16,10 @@ import { policyFor, policyProblem, providerPreset, + PLATFORM_PROFILE, + profileProblem, withProfile, + withRenamedProfile, withoutProfile, type ProviderId, type ReplicationChoice, @@ -27,8 +30,8 @@ import { errText } from '@/lib/storage/util' import { useStorageConfig } from '@/hooks/use-storage-config' import { Button } from '@/components/ui/button' import { ErrorState, LoadingBlock } from '@/components/ui/states' -import { ProfileForm } from '@/components/storage/profile-form' -import { StorageTest } from '@/components/storage/storage-test' +import { ProfileForm, initialDraft } from '@/components/storage/profile-form' +import { StorageTest, profileSnapshot } from '@/components/storage/storage-test' import { CostCard } from '@/components/storage/cost-card' import { cn } from '@/lib/utils' import { formatDate } from '@/lib/view/format' @@ -44,12 +47,12 @@ const TILE_ICON: Readonly> = { } export function StorageWizard(): JSX.Element { - const { config, loading, error, storable, save, reload } = useStorageConfig() + const { config, loading, error, storable, save, reload, discard } = useStorageConfig() const [provider, setProvider] = useState(null) const [editing, setEditing] = useState(null) if (loading && !config) return - if (error) return + if (error) return if (!config) return const start = (p: ProviderId, existing: StorageProfile | null = null): void => { @@ -123,6 +126,37 @@ export function StorageWizard(): JSX.Element { ) } +/** + * The stored settings could not be opened (sealed under an earlier key, or unreadable). Offer a + * retry and, since nothing else can open them, a way to discard them and start again. + */ +function UnreadableSettings({ message, onRetry, onDiscard }: { message: string; onRetry: () => void; onDiscard: () => Promise }): JSX.Element { + const [err, setErr] = useState(null) + return ( +

    + +
    + If they were sealed with an earlier key, they cannot be opened any more. + +
    + {err ?

    {err}

    : null} +
    + ) +} + function AddProfile({ provider, existing, @@ -139,10 +173,17 @@ function AddProfile({ onDone: () => void }): JSX.Element { const preset = providerPreset(provider) - const [draft, setDraft] = useState<{ profile: StorageProfile; problem: string | null } | null>(null) - const [tested, setTested] = useState(null) + // An edit starts from the saved profile, so it can be tested before any field changes. + const [draft, setDraft] = useState<{ profile: StorageProfile; problem: string | null } | null>(() => { + if (existing === null || provider === 'platform') return null + const profile = initialDraft(provider, existing) + return { profile, problem: profileProblem(profile) } + }) + // A test result counts only for the exact values it ran against. + const [tested, setTested] = useState<{ ok: boolean; snapshot: string } | null>(null) const [saving, setSaving] = useState(false) const [saveError, setSaveError] = useState(null) + const statusId = useId() if (provider === 'platform') { const has = config.profiles.some((p) => p.settings.kind === 'platform') @@ -161,7 +202,7 @@ function AddProfile({ onClick={async () => { setSaving(true) try { - await save(withProfile(config, { name: 'platform', settings: { kind: 'platform', provider: 'platform' }, secrets: {} })) + await save(withProfile(config, { name: PLATFORM_PROFILE, settings: { kind: 'platform', provider: 'platform' }, secrets: {} })) onDone() } catch (e) { setSaveError(errText(e)) @@ -182,6 +223,10 @@ function AddProfile({ const nameTaken = draft !== null && existing?.name !== draft.profile.name && config.profiles.some((p) => p.name === draft.profile.name) const problem = draft?.problem ?? (draft ? null : 'fill in the fields') const blocking = problem ?? (nameTaken ? 'a profile with that name exists' : null) + const snapshot = draft ? profileSnapshot(draft.profile) : '' + const result = tested !== null && tested.snapshot === snapshot ? tested.ok : null + const status = + blocking ?? (!storable ? 'Sign in with a stored key to save.' : result === null ? 'Run the test first.' : result ? 'All checks passed.' : 'Some checks failed: you can still save, but browser pushes need them all.') return (
    @@ -192,27 +237,25 @@ function AddProfile({ { - setDraft({ profile, problem: p }) - setTested(null) - }} + onChange={(profile, p) => setDraft({ profile, problem: p })} /> - {blocking && draft ?

    {blocking}

    : null} - {draft && !blocking ? : null} + {draft && !blocking ? ( + setTested({ ok, snapshot: snap })} /> + ) : null}
    - - {tested === null ? 'Run the test first.' : tested ? 'All checks passed.' : 'Some checks failed: you can still save, but browser pushes need them all.'} + + {status}
    {saveError ?

    {saveError}

    : null} @@ -235,8 +278,22 @@ function AddProfile({ } function Profiles({ config, storable, save, onEdit }: { config: StorageConfig; storable: boolean; save: (c: StorageConfig) => Promise; onEdit: (p: StorageProfile) => void }): JSX.Element { + const [error, setError] = useState(null) + const remove = async (name: string): Promise => { + setError(null) + try { + await save(withoutProfile(config, name)) + } catch (e) { + setError(`Could not remove ${name}: ${errText(e)}`) + } + } return (
    + {error ? ( +

    + {error} +

    + ) : null}

    Your storage

    @@ -272,7 +329,7 @@ function Profiles({ config, storable, save, onEdit }: { config: StorageConfig; s disabled={!storable} aria-label={`Remove ${p.name}`} onClick={() => { - if (window.confirm(`Remove ${p.name} from this browser? Packs already stored there stay; repos that point at it will ask for another place on their next browser push.`)) void save(withoutProfile(config, p.name)) + if (window.confirm(`Remove ${p.name} from this browser? Packs already stored there stay; repos that point at it will ask for another place on their next browser push.`)) void remove(p.name) }} > Remove @@ -363,8 +420,9 @@ function DefaultPolicy({ config, storable, save }: { config: StorageConfig; stor > Save default - {problem && targets.length > 0 ? {problem} : null} - {saved ? Saved. : null} + + {saved ? 'Saved.' : problem && targets.length > 0 ? problem : ''} + {err ? {err} : null}
diff --git a/forge-web/contexts/auth-context.tsx b/forge-web/contexts/auth-context.tsx index 961254fd1..a6863b7e0 100644 --- a/forge-web/contexts/auth-context.tsx +++ b/forge-web/contexts/auth-context.tsx @@ -105,6 +105,12 @@ export function AuthProvider({ useEffect(() => controller.subscribe(setState), [controller]) + // A notice from a sign-in step (e.g. storage settings that could not survive a key renewal). + const notice = state.notice ?? null + useEffect(() => { + if (notice) toast({ title: 'Storage settings were not carried over', tone: 'warn', detail: notice }) + }, [notice]) + const [vaults, setVaults] = useState([]) const reloadVaults = useCallback(() => { controller.storedVaults().then(setVaults, () => setVaults([])) diff --git a/forge-web/e2e/storage-wizard.spec.ts b/forge-web/e2e/storage-wizard.spec.ts index 2c4f0604c..dd24f6d51 100644 --- a/forge-web/e2e/storage-wizard.spec.ts +++ b/forge-web/e2e/storage-wizard.spec.ts @@ -10,8 +10,9 @@ import { E2E_DEVNET, idFile, runAxe, shot, signedIn, stateFile, unlock } from '. * docker compose -f infra/docker-compose.yml up -d minio minio-init static-http * E2E_DEVNET=moutai E2E_WRITE=1 pnpm exec playwright test storage-wizard.spec.ts * - * s1. every row of the browser test passes on MinIO; the profile is saved into the vault, never - * into localStorage; it survives a reload + unlock; a default policy is set. + * s1. on MinIO every write, read, range and CORS row passes, and the public-read row fails with + * the reason a loopback address cannot be published; the profile is saved into the vault, + * never into localStorage; it survives a reload + unlock; a default policy is set. * s2. a reachable host with no CORS (the static nginx) fails the CORS rows and shows the * copy-paste fix, prefilled with the bucket and this origin. * s3. a repo's own policy (its Settings → Your browser pushes). @@ -51,14 +52,19 @@ test('s1. MinIO passes every browser check, and the profile is sealed in the vau await page.getByTestId('tile-minio').click() await fillS3(page, { name: 'minio-e2e', ...MINIO }) await page.getByRole('button', { name: /^test$/i }).click() - for (const row of ['put', 'get', 'public', 'cors-range', 'cors-put', 'delete']) { + for (const row of ['put', 'get', 'range', 'cors-put', 'delete']) { await expect(page.getByTestId(`probe-${row}`)).toHaveAttribute('data-state', 'ok', { timeout: 60_000 }) } + // The anonymous read works from here, but 127.0.0.1 is not an address anyone else can read: + // it would be recorded on chain, so the row fails and says why (uploads refuse it too). + const publicRow = page.getByTestId('probe-public') + await expect(publicRow).toHaveAttribute('data-state', 'fail') + await expect(publicRow).toContainText(/Readable from here, but 127\.0\.0\.1:9000 is only reachable from this machine/) await expect(page.getByTestId('cors-fix')).toHaveCount(0) - await shot(page, 'a-02-minio-all-passed') + await shot(page, 'a-02-minio-tested') await page.getByRole('button', { name: /save profile/i }).click() await expect(page.getByTestId('profile-list')).toContainText('minio-e2e') - await expect(page.getByTestId('profile-list')).toContainText('passed') + await expect(page.getByTestId('profile-list')).toContainText('failed some checks') // Default policy: this profile, one copy. await page.getByRole('checkbox', { name: 'minio-e2e' }).first().check() diff --git a/forge-web/hooks/use-storage-config.ts b/forge-web/hooks/use-storage-config.ts index 7e877d57e..881094c38 100644 --- a/forge-web/hooks/use-storage-config.ts +++ b/forge-web/hooks/use-storage-config.ts @@ -12,7 +12,7 @@ import { useCallback } from 'react' import { useAuth } from '@/contexts/auth-context' import { useAsync } from '@/hooks/use-async' import { DEFAULT_NETWORK } from '@/lib/constants' -import { EMPTY_STORAGE_CONFIG, loadStorageConfig, saveStorageConfig, type StorageConfig } from '@/lib/storage' +import { EMPTY_STORAGE_CONFIG, discardStorageConfig, loadStorageConfig, saveStorageConfig, type StorageConfig } from '@/lib/storage' export interface StorageConfigState { readonly config: StorageConfig | null @@ -22,6 +22,8 @@ export interface StorageConfigState { readonly storable: boolean save: (next: StorageConfig) => Promise reload: () => void + /** Delete stored settings this key cannot open, then reload (empty). */ + discard: () => Promise } export function useStorageConfig(): StorageConfigState { @@ -41,5 +43,10 @@ export function useStorageConfig(): StorageConfigState { }, [identity, reload, storable], ) - return { config: identity === null ? null : state.data, loading: state.loading, error: state.error, storable, save, reload } + const discard = useCallback(async (): Promise => { + if (identity === null) return + await discardStorageConfig(DEFAULT_NETWORK, identity) + reload() + }, [identity, reload]) + return { config: identity === null ? null : state.data, loading: state.loading, error: state.error, storable, save, reload, discard } } diff --git a/forge-web/lib/auth/controller.ts b/forge-web/lib/auth/controller.ts index 25f479091..bea56f125 100644 --- a/forge-web/lib/auth/controller.ts +++ b/forge-web/lib/auth/controller.ts @@ -72,6 +72,8 @@ export interface AuthState { readonly session: AuthSession | null readonly isLoading: boolean readonly error: string | null + /** Something the user should know after a sign-in step succeeded (null when nothing). */ + readonly notice?: string | null } type Listener = (state: AuthState) => void @@ -230,7 +232,12 @@ export class AuthController { */ private async adopt(identityId: string, key: LimitedKey, protection: Protection): Promise { const secret: VaultSecret = { identityId, keyId: key.keyId, wif: key.wif } - await storeInVault(this.network, secret, protection) + const { storageSettingsDropped } = await storeInVault(this.network, secret, protection) + if (storageSettingsDropped) { + this.setState({ + notice: 'Your storage settings were sealed with the previous key, which was locked when you renewed it, so they could not be carried over. Add your storage again in Settings → Storage.', + }) + } try { return await this.open(secret, 'vault', key.limits) } catch (e) { diff --git a/forge-web/lib/auth/vault.ts b/forge-web/lib/auth/vault.ts index d78e692d7..32f032b0d 100644 --- a/forge-web/lib/auth/vault.ts +++ b/forge-web/lib/auth/vault.ts @@ -28,7 +28,7 @@ import { hkdf } from '@noble/hashes/hkdf.js' import { sha256 } from '@noble/hashes/sha2.js' import type { Network } from '../constants' -import { idbDelete, idbEntries, idbGet, idbPut } from '../idb' +import { idbBatch, idbDelete, idbEntries, idbGet, idbPut } from '../idb' /** Unlocked vaults lock themselves after this long (spec §2.3). */ export const AUTO_LOCK_MS = 12 * 60 * 60 * 1000 @@ -289,11 +289,22 @@ export interface Protection { readonly passkey?: { credentialId: Uint8Array; prfSalt: Uint8Array; output: Uint8Array } } +/** What {@link storeInVault} did with storage settings sealed under the record it replaced. */ +export interface StoreOutcome { + /** + * Storage settings existed but could not be carried across (the vault was locked when the + * key was renewed, so they could not be opened). They are deleted; the user must add them + * again, and the UI says so. + */ + readonly storageSettingsDropped: boolean +} + /** * Seal `secret` into the vault for `network`, replacing any earlier record for its identity, - * and keep it unlocked for this session. At least one protection is required. + * and keep it unlocked for this session. At least one protection is required. The record and + * the (re-sealed or deleted) storage settings are written in one transaction. */ -export async function storeInVault(network: Network, secret: VaultSecret, protection: Protection): Promise { +export async function storeInVault(network: Network, secret: VaultSecret, protection: Protection): Promise { assertDedicatedOrigin() if (!protection.passphrase && !protection.passkey) throw new Error('the vault needs a passphrase or a passkey') if (protection.passphrase !== undefined && protection.passphrase.length < MIN_PASSPHRASE) { @@ -302,7 +313,8 @@ export async function storeInVault(network: Network, secret: VaultSecret, protec const { identityId } = secret // A renewal replaces the record (and its data key): carry the storage settings across when // this session can open them; otherwise they cannot be opened any more and are dropped. - const carried = await readStorageBlob(network, identityId).catch(() => null) + const hadBlob = (await idbGet('vault', storageBlobKey(network, identityId))) !== undefined + const carried = hadBlob ? await readStorageBlob(network, identityId).catch(() => null) : null const dataKey = random(32) let storageKey: CryptoKey try { @@ -332,13 +344,26 @@ export async function storeInVault(network: Network, secret: VaultSecret, protec slots, } storageKey = await deriveStorageKey(dataKey, network, identityId) - await idbPut('vault', key(network, identityId), record) - if (carried !== null) await idbPut('vault', storageBlobKey(network, identityId), await sealBlob(storageKey, network, identityId, carried)) - else await idbDelete('vault', storageBlobKey(network, identityId)) + const blob = carried !== null ? await sealBlob(storageKey, network, identityId, carried) : undefined + // One transaction: a crash between the two writes must not leave settings sealed under a + // data key no record holds any more. + await idbBatch('vault', [ + [key(network, identityId), record], + [storageBlobKey(network, identityId), blob], + ]) } finally { dataKey.fill(0) } setUnlocked(network, secret, storageKey) + return { storageSettingsDropped: hadBlob && carried === null } +} + +/** + * Delete storage settings that cannot be opened (sealed under an earlier key, or unreadable), + * so the user can start over instead of being stuck on an error. + */ +export async function discardStorageBlob(network: Network, identityId: string): Promise { + await idbDelete('vault', storageBlobKey(network, identityId)) } /** diff --git a/forge-web/lib/idb.ts b/forge-web/lib/idb.ts index def2802ac..39caaac84 100644 --- a/forge-web/lib/idb.ts +++ b/forge-web/lib/idb.ts @@ -85,6 +85,33 @@ export async function idbPut(store: StoreName, key: string, value: T): Promis await run(store, 'readwrite', (s) => s.put(value, key)) } +/** + * Several puts and deletes in ONE store, atomically: one transaction, so a crash or a failed + * write leaves either all of them or none (`value: undefined` deletes the key). + */ +export async function idbBatch(store: StoreName, ops: readonly (readonly [string, unknown])[]): Promise { + if (!hasIndexedDb()) { + const m = mem(store) + for (const [k, v] of ops) { + if (v === undefined) m.delete(k) + else m.set(k, structuredClone(v)) + } + return + } + const db = await open() + await new Promise((resolve, reject) => { + const tx = db.transaction(store, 'readwrite') + const s = tx.objectStore(store) + for (const [k, v] of ops) { + if (v === undefined) s.delete(k) + else s.put(v, k) + } + tx.oncomplete = () => resolve() + tx.onerror = () => reject(tx.error ?? new Error('IndexedDB transaction failed')) + tx.onabort = () => reject(tx.error ?? new Error('IndexedDB transaction aborted')) + }) +} + /** Delete one value (no-op when absent). */ export async function idbDelete(store: StoreName, key: string): Promise { if (!hasIndexedDb()) { diff --git a/forge-web/lib/net.ts b/forge-web/lib/net.ts new file mode 100644 index 000000000..1d6899351 --- /dev/null +++ b/forge-web/lib/net.ts @@ -0,0 +1,52 @@ +/** + * Which URLs may be recorded on chain, and which a reader may fetch. + * + * Anything a `packManifest` records is read by every visitor's browser, so it must be a public + * https URL: never plain http, never this machine or a private network. Otherwise one owner's + * manifest could make every reader's browser send requests to its own local services (blind + * GET-CSRF, port probing), and a copy only its uploader can reach would count as "stored". + * Loopback http is fine only for what the user's own browser talks to directly while setting + * up storage (a local kubo API, a local MinIO endpoint), never for what gets published. + */ + +/** + * Whether this build may talk plain http to this machine (a local MinIO or kubo): devnet and + * dev builds only. The CSP (`app/layout.tsx`) and the storage profile checks both follow it, + * so a production build never opens loopback http for any page. + */ +export const LOCAL_HTTP_ALLOWED: boolean = process.env.NEXT_PUBLIC_NETWORK === 'devnet' || process.env.NEXT_PUBLIC_FORGE_DEV === '1' + +/** Whether `hostname` names this machine or a private / link-local network. */ +export function isPrivateHost(hostname: string): boolean { + const h = hostname.toLowerCase().replace(/^\[|\]$/g, '') + if (h === 'localhost' || h.endsWith('.localhost') || h.endsWith('.local') || h.endsWith('.internal')) return true + const v4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(h) + if (v4) { + const [a, b] = [Number(v4[1]), Number(v4[2])] + return a === 0 || a === 10 || a === 127 || (a === 100 && b >= 64 && b <= 127) || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) + } + if (h.includes(':')) { + // IPv6: loopback, unspecified, unique-local fc00::/7, link-local fe80::/10, v4-mapped. + return h === '::1' || h === '::' || /^f[cd]/.test(h) || /^fe[89ab]/.test(h) || h.startsWith('::ffff:') + } + return false +} + +/** Whether `url` is a public https URL (what readers may fetch and a manifest may record). */ +export function isPublicHttpsUrl(url: string): boolean { + try { + const u = new URL(url) + return u.protocol === 'https:' && !isPrivateHost(u.hostname) && u.username === '' && u.password === '' + } catch { + return false + } +} + +/** + * Whether a manifest may record `uri`: a public https URL, or a non-HTTP locator browsers do + * not fetch directly (`ipfs://`, `s3://`, `platform://`). + */ +export function isRecordableUri(uri: string): boolean { + if (/^(ipfs|s3|platform):\/\//i.test(uri)) return true + return isPublicHttpsUrl(uri) +} diff --git a/forge-web/lib/storage/cors.ts b/forge-web/lib/storage/cors.ts index e846dd6ba..48da82376 100644 --- a/forge-web/lib/storage/cors.ts +++ b/forge-web/lib/storage/cors.ts @@ -10,6 +10,7 @@ * because not every provider accepts a wildcard there. */ +import { FORGE_RPC_PATHS } from './ipfs' import type { ProviderId } from './profiles' /** The headers a signed browser request sends that are not CORS-safelisted. */ @@ -58,13 +59,15 @@ export function corsFix(provider: ProviderId, bucket: string, origin: string): C } case 'minio': return { - where: 'MinIO answers CORS for every origin by default. If it was restricted, allow this app, then make the bucket publicly readable (not writable):', - text: [`mc admin config set api cors_allow_origin="${origin}"`, 'mc admin service restart ', `mc anonymous set download /${b}`].join('\n'), + where: + 'MinIO answers CORS for every origin by default (cors_allow_origin="*", which reads need: every Forge web app and mirror reads the objects). If it was restricted, restore it, then make the bucket publicly readable (not writable):', + text: ['mc admin config set api cors_allow_origin="*"', 'mc admin service restart ', `mc anonymous set download /${b}`].join('\n'), } case 'kubo': case 'pinning': return { - where: 'kubo refuses RPC calls from web pages unless their origin is allowed. Run, then restart the daemon:', + where: + 'kubo refuses RPC calls from web pages unless their origin is allowed. The RPC API is the node’s admin interface, so give this app a token limited to what Forge calls (below), put that token in the profile’s API Authorization field, and add this origin to the allowed list. Run, then restart the daemon:', text: kuboCorsLines(origin), } case 'platform': @@ -72,12 +75,23 @@ export function corsFix(provider: ProviderId, bucket: string, origin: string): C } } -/** `ipfs config` lines allowing this app to call the RPC API and read the gateway. */ -function kuboCorsLines(origin: string): string { +/** + * Shell lines that (1) create a kubo API token limited to the paths Forge calls, (2) ADD this + * app's origin to the RPC API's allowed origins without dropping existing ones (IPFS WebUI's, + * for instance), and (3) let any origin read the gateway. `jq` merges the list. + */ +export function kuboCorsLines(origin: string): string { + const paths = JSON.stringify(FORGE_RPC_PATHS) return [ - `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin '["${origin}"]'`, + '# 1. A token that can only add, pin-check, unpin and identify (not the whole admin API):', + `TOKEN=$(openssl rand -hex 24)`, + `ipfs config --json API.Authorizations.dash-forge "{\\"AuthSecret\\": \\"bearer:$TOKEN\\", \\"AllowedPaths\\": ${paths.replace(/"/g, '\\"')}}"`, + 'echo "API Authorization for the profile: Bearer $TOKEN"', + '# 2. Allow this app (keeping the origins already allowed):', + `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin "$(ipfs config API.HTTPHeaders.Access-Control-Allow-Origin 2>/dev/null | jq -c '(. // []) + ["${origin}"] | unique' || echo '["${origin}"]')"`, `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Methods '["POST"]'`, `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Headers '["Authorization"]'`, + '# 3. Let any origin read the gateway (the content is public):', `ipfs config --json Gateway.HTTPHeaders.Access-Control-Allow-Origin '["*"]'`, `ipfs config --json Gateway.HTTPHeaders.Access-Control-Expose-Headers '["Content-Range", "Content-Length", "ETag"]'`, ].join('\n') diff --git a/forge-web/lib/storage/ipfs.ts b/forge-web/lib/storage/ipfs.ts index 4dc2dbc3c..caea9ffdb 100644 --- a/forge-web/lib/storage/ipfs.ts +++ b/forge-web/lib/storage/ipfs.ts @@ -6,10 +6,16 @@ * * kubo's RPC API refuses cross-origin calls unless `API.HTTPHeaders.Access-Control-Allow-Origin` * lists this origin (it answers 403 otherwise); the wizard prints the `ipfs config` lines. + * + * The RPC API is kubo's ADMIN interface. Allowing a web origin to call it hands that origin + * the node, unless kubo's `API.Authorizations` limits the token it uses to the few paths Forge + * needs ({@link FORGE_RPC_PATHS}); the wizard says so and prints that configuration, and the + * token goes into the profile's `apiAuth`. */ import { cidV1RawLeaves, isCid } from './cid' import type { ProfilePublic, ProfileSecrets } from './profiles' +import { TimeoutError, isHeaderSafe, timedFetch } from './util' export type IpfsSettings = Extract @@ -17,7 +23,9 @@ export type IpfsSettings = Extract { +async function rpc(s: IpfsSettings, secrets: ProfileSecrets, op: string, pathAndQuery: string, body?: FormData, uploadBytes = 0): Promise { const headers = new Headers() - if (secrets.apiAuth) headers.set('authorization', secrets.apiAuth) + if (secrets.apiAuth) { + // A value fetch refuses would be quoted in its error: refuse it here, without echoing it. + if (!isHeaderSafe(secrets.apiAuth)) throw new IpfsError(op, 'the API Authorization value holds characters an HTTP header cannot carry; paste it again') + headers.set('authorization', secrets.apiAuth) + } const url = `${s.api.replace(/\/+$/, '')}/api/v0/${pathAndQuery}` + let text: string let resp: Response try { - resp = await fetch(url, { method: 'POST', headers, ...(body ? { body } : {}), redirect: 'error', credentials: 'omit', signal: AbortSignal.timeout(RPC_TIMEOUT_MS) }) + const r = await timedFetch(url, { method: 'POST', headers, ...(body ? { body } : {}), redirect: 'error', credentials: 'omit' }, { uploadBytes }) + resp = r.resp + text = await r.text() } catch (e) { - throw new IpfsError(op, `${op}: the kubo API at ${new URL(url).host} is unreachable or refused this origin (CORS)${e instanceof Error ? ` (${e.message})` : ''}`) + const why = e instanceof TimeoutError ? e.message : 'unreachable, or it refused this origin (CORS)' + throw new IpfsError(op, `${op}: the kubo API at ${new URL(url).host}: ${why}`) } - const text = await resp.text() if (!resp.ok) { const msg = /"Message"\s*:\s*"([^"]{1,200})"/.exec(text)?.[1] ?? text.slice(0, 200) - const hint = resp.status === 403 ? ' — kubo refuses this origin: allow it in API.HTTPHeaders (see the fix below)' : '' + const hint = + resp.status === 403 + ? ' — kubo refuses this origin: allow it in API.HTTPHeaders (see the fix below)' + : resp.status === 401 + ? ' — kubo wants an API Authorization token (see API.Authorizations)' + : '' throw new IpfsError(op, `${op} failed: HTTP ${resp.status} ${msg}${hint}`) } return text @@ -66,7 +86,7 @@ export async function kuboVersion(s: IpfsSettings, secrets: ProfileSecrets): Pro export async function addVerified(s: IpfsSettings, secrets: ProfileSecrets, bytes: Uint8Array): Promise { const form = new FormData() form.append('file', new Blob([new Uint8Array(bytes)]), 'pack') - const text = await rpc(s, secrets, 'ipfs add', `add?${ADD_PARAMS}`, form) + const text = await rpc(s, secrets, 'ipfs add', `add?${ADD_PARAMS}`, form, bytes.length) const line = text.trim().split('\n').pop() ?? '' const cid = /"Hash"\s*:\s*"([^"]+)"/.exec(line)?.[1] ?? '' const expected = cidV1RawLeaves(bytes) @@ -103,24 +123,32 @@ interface PinStatus { async function psa(s: IpfsSettings, secrets: ProfileSecrets, op: string, path: string, init: RequestInit = {}): Promise { if (!secrets.pinningToken) throw new IpfsError(op, 'the pinning service needs an access token') + if (!isHeaderSafe(secrets.pinningToken)) throw new IpfsError(op, 'the pinning token holds characters an HTTP header cannot carry; paste it again') const url = `${s.pinningEndpoint.replace(/\/+$/, '')}${path}` let resp: Response + let text: string try { - resp = await fetch(url, { + const r = await timedFetch(url, { ...init, headers: { authorization: `Bearer ${secrets.pinningToken}`, 'content-type': 'application/json' }, redirect: 'error', credentials: 'omit', - signal: AbortSignal.timeout(RPC_TIMEOUT_MS), }) + resp = r.resp + text = await r.text() } catch (e) { - throw new IpfsError(op, `${op}: the pinning service is unreachable or refused this origin (CORS)${e instanceof Error ? ` (${e.message})` : ''}`) + const why = e instanceof TimeoutError ? e.message : 'unreachable, or it refused this origin (CORS)' + throw new IpfsError(op, `${op}: the pinning service: ${why}`) } if (!resp.ok) { const hint = resp.status === 401 || resp.status === 403 ? ' — check the access token' : '' throw new IpfsError(op, `${op} failed: HTTP ${resp.status}${hint}`) } - return resp.json() as Promise + try { + return JSON.parse(text) as unknown + } catch { + throw new IpfsError(op, `${op}: the pinning service did not answer with JSON`) + } } function pinStatus(v: unknown): PinStatus { diff --git a/forge-web/lib/storage/probe.ts b/forge-web/lib/storage/probe.ts index 8d64135cd..97278daa8 100644 --- a/forge-web/lib/storage/probe.ts +++ b/forge-web/lib/storage/probe.ts @@ -2,23 +2,27 @@ * The storage wizard's live test (`ux-dx-spec.md` §3.1 step 3), run from this page, so it * checks exactly what the browser will do later — including CORS, which a CLI test cannot see. * - * S3 rows: signed PUT → signed GET → anonymous GET via the public URL → CORS for ranged reads - * (GET + Range, `Content-Range` visible) → CORS for browser pushes (PUT) → delete probe. - * IPFS rows: kubo API → add + CID + pin → gateway re-read (Range) → pinning service → unpin. + * S3 rows: signed PUT → signed GET → anonymous GET via the public URL → ranged read with + * `Content-Range` visible → CORS for browser pushes (PUT) → delete probe. + * IPFS rows: kubo API → add + CID + pin → gateway re-read (Range) → public gateway → + * pinning service → unpin. * - * A page cannot see WHY a cross-origin request failed: a CORS refusal and a dead host both - * reject `fetch`. {@link reachable} tells them apart with an opaque `no-cors` request, which - * resolves whenever the host answers at all, so a red row says "blocked by CORS" only when the - * host is up. + * A page cannot see WHY a cross-origin request failed: a CORS refusal, a refused redirect and a + * dead host all reject `fetch`. {@link reachable} tells a dead host apart with an opaque + * `no-cors` request to the same origin, which resolves whenever the host answers at all; a + * `manual`-redirect probe then tells a redirect (wrong region or endpoint) from CORS. + * + * The public address is what everyone else reads, and it is recorded on chain: a row fails + * when it is only reachable from this machine or its network, even if it answers here. */ -import { S3Error, deleteObject, getObject, publicObjectUrl, putObject, type S3Settings } from './s3' +import { S3Error, deleteObject, getObject, getPublic, objectUrl, publicObjectUrl, putObject, type S3Settings } from './s3' import { IpfsError, addVerified, gatewayUrl, kuboVersion, pinningReachable, unpin, type IpfsSettings } from './ipfs' -import { normalizedPrefix, type ProfileSecrets, type StorageProfile } from './profiles' +import { normalizedPrefix, publishProblem, type ProfileSecrets, type StorageProfile } from './profiles' import { sha256Hex } from './sigv4' -import { bytesEqual, errText } from './util' +import { bytesEqual, errText, timedFetch } from './util' -export type RowId = 'put' | 'get' | 'public' | 'cors-range' | 'cors-put' | 'delete' | 'api' | 'add' | 'gateway' | 'pinning' | 'unpin' +export type RowId = 'put' | 'get' | 'public' | 'range' | 'cors-put' | 'delete' | 'api' | 'add' | 'gateway' | 'public-gateway' | 'pinning' | 'unpin' export type RowState = 'pending' | 'running' | 'ok' | 'fail' | 'skipped' /** One row of the live test. `cors` marks a failure the CORS fix block addresses. */ @@ -34,7 +38,7 @@ export const S3_ROWS: readonly { id: RowId; label: string }[] = [ { id: 'put', label: 'signed PUT' }, { id: 'get', label: 'signed GET' }, { id: 'public', label: 'anonymous GET via public URL' }, - { id: 'cors-range', label: 'CORS preflight (GET, Range)' }, + { id: 'range', label: 'ranged read (Content-Range visible)' }, { id: 'cors-put', label: 'CORS preflight (PUT) for browser pushes' }, { id: 'delete', label: 'delete probe' }, ] @@ -43,20 +47,37 @@ export const IPFS_ROWS: readonly { id: RowId; label: string }[] = [ { id: 'api', label: 'kubo API' }, { id: 'add', label: 'add + CID check + pin' }, { id: 'gateway', label: 'gateway re-read (Range)' }, + { id: 'public-gateway', label: 'anonymous read via public gateway' }, { id: 'pinning', label: 'pinning service' }, { id: 'unpin', label: 'unpin probe' }, ] -/** Whether `url`'s host answers at all (an opaque no-cors request; never readable). */ -async function reachable(url: string): Promise { +/** Whether `url`'s origin answers at all (an opaque no-cors request; never readable). */ +async function reachable(url: string | URL): Promise { try { - await fetch(url, { mode: 'no-cors', credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(10_000) }) + await fetch(new URL(url).origin, { mode: 'no-cors', credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(10_000) }) return true } catch { return false } } +/** Whether `url` answers with a redirect (opaque, so only its type is visible). */ +async function redirects(url: string | URL): Promise { + try { + const r = await fetch(url, { mode: 'no-cors', redirect: 'manual', credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(10_000) }) + return r.type === 'opaqueredirect' + } catch { + return false + } +} + +/** Why a request to `url` never got an answer: down, redirected elsewhere, or CORS. */ +async function whyBlocked(url: string | URL): Promise<'down' | 'redirect' | 'cors'> { + if (!(await reachable(url))) return 'down' + return (await redirects(url)) ? 'redirect' : 'cors' +} + /** A unique probe body, so a stale cached object can never pass for this run's upload. */ function probeBody(): Uint8Array { const nonce = Array.from(crypto.getRandomValues(new Uint8Array(12)), (b) => b.toString(16).padStart(2, '0')).join('') @@ -66,27 +87,36 @@ function probeBody(): Uint8Array { type Report = (id: RowId, state: RowState, detail: string, cors?: boolean) => void /** Run the S3 test, reporting each row as it settles. */ -async function probeS3(s: S3Settings, secrets: ProfileSecrets, report: Report): Promise { +async function probeS3(p: StorageProfile & { settings: S3Settings }, report: Report): Promise { + const s = p.settings + const secrets = p.secrets const body = probeBody() const key = `${normalizedPrefix(s.prefix)}probe/forge-web-test-${await sha256Hex(body)}.txt` + const apiUrl = objectUrl(s, key) // 1 + 5: a PUT that lands proves the PUT preflight passed; one refused before any answer is - // CORS when the endpoint is reachable. + // CORS when the bucket's host answers and does not redirect. report('put', 'running', '') try { await putObject(s, secrets, key, body, 'text/plain') report('put', 'ok', `wrote ${key}`) report('cors-put', 'ok', 'the browser was allowed to send a signed PUT from this origin') } catch (e) { - const blocked = e instanceof S3Error && e.status === 0 && (await reachable(s.endpoint)) - if (blocked) { + const why = e instanceof S3Error && e.status === 0 ? await whyBlocked(apiUrl) : null + if (why === 'cors') { report('put', 'fail', 'the browser refused to send it (see the CORS row)') report('cors-put', 'fail', 'the bucket’s CORS rules do not allow a signed PUT from this origin', true) + } else if (why === 'redirect') { + report('put', 'fail', `${apiUrl.host} answered with a redirect: the bucket lives in another region or behind another endpoint; set that endpoint`) + report('cors-put', 'skipped', 'needs a PUT that reaches the bucket') + } else if (why === 'down') { + report('put', 'fail', `${apiUrl.host} did not answer (check the endpoint${s.pathStyle ? '' : ', and that the bucket subdomain resolves'})`) + report('cors-put', 'skipped', 'needs a PUT that reaches the bucket') } else { report('put', 'fail', errText(e)) report('cors-put', 'skipped', 'needs a PUT that reaches the bucket') } - for (const id of ['get', 'public', 'cors-range', 'delete'] as const) report(id, 'skipped', 'needs the probe object') + for (const id of ['get', 'public', 'range', 'delete'] as const) report(id, 'skipped', 'needs the probe object') return } @@ -96,38 +126,45 @@ async function probeS3(s: S3Settings, secrets: ProfileSecrets, report: Report): if (bytesEqual(got, body)) report('get', 'ok', 'read back byte for byte') else report('get', 'fail', 'the bucket returned different bytes (a cache or proxy in front of it?)') } catch (e) { - report('get', 'fail', errText(e), e instanceof S3Error && e.status === 0) + report('get', 'fail', errText(e)) } const publicUrl = publicObjectUrl(s, key) - report('public', 'running', '') const publicHost = new URL(publicUrl).host + const unpublishable = publishProblem(p) + report('public', 'running', '') + // Whether an anonymous read worked from here (the range row builds on it), apart from whether + // the address is one others can use (a local MinIO answers here and nowhere else). let publicOk = false try { - const resp = await fetch(publicUrl, { credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(20_000) }) - const got = new Uint8Array(await resp.arrayBuffer()) - if (!resp.ok) report('public', 'fail', `HTTP ${resp.status} from ${publicHost}: the objects are not publicly readable`) - else if (!bytesEqual(got, body)) report('public', 'fail', `${publicHost} served different bytes (check the public URL)`) + const got = await getPublic(s, key) + if (got.status !== 200) report('public', 'fail', `HTTP ${got.status} from ${publicHost}: the objects are not publicly readable`) + else if (!bytesEqual(got.bytes, body)) report('public', 'fail', `${publicHost} served different bytes (check the public URL)`) else { publicOk = true - report('public', 'ok', `anonymous GET ${publicHost}`) + if (unpublishable) report('public', 'fail', `Readable from here, but ${unpublishable}`) + else report('public', 'ok', `anonymous GET ${publicHost}`) } } catch { - const up = await reachable(publicUrl) - report('public', 'fail', up ? 'the browser was not allowed to read it: no Access-Control-Allow-Origin on GET' : `${publicHost} did not answer`, up) + const why = await whyBlocked(publicUrl) + report( + 'public', + 'fail', + why === 'cors' ? 'the browser was not allowed to read it: no Access-Control-Allow-Origin on GET' : why === 'redirect' ? `${publicHost} redirected the read elsewhere` : `${publicHost} did not answer`, + why === 'cors', + ) } - report('cors-range', 'running', '') - if (!publicOk) report('cors-range', 'skipped', 'needs a public read') + report('range', 'running', '') + if (!publicOk) report('range', 'skipped', 'needs a public read') else { try { - const resp = await fetch(publicUrl, { headers: { Range: 'bytes=0-9' }, credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(20_000) }) - const range = resp.headers.get('content-range') - if (resp.status !== 206) report('cors-range', 'fail', `a ranged GET returned ${resp.status} instead of 206: browsing needs HTTP Range support on the public URL`) - else if (range === null) report('cors-range', 'fail', 'Content-Range is not exposed to scripts (Access-Control-Expose-Headers)', true) - else report('cors-range', 'ok', 'Range allowed, Content-Range exposed') + const got = await getPublic(s, key, 'bytes=0-9') + if (got.status !== 206) report('range', 'fail', `a ranged GET returned ${got.status} instead of 206: browsing needs HTTP Range support on the public URL`) + else if (got.contentRange === null) report('range', 'fail', 'Content-Range is not exposed to scripts (Access-Control-Expose-Headers)', true) + else report('range', 'ok', 'Range honoured, Content-Range exposed') } catch { - report('cors-range', 'fail', 'the CORS preflight for a Range request was refused: allow the Range request header', true) + report('range', 'fail', 'the ranged read was refused: allow the Range request header in the bucket’s CORS rules', true) } } @@ -136,19 +173,36 @@ async function probeS3(s: S3Settings, secrets: ProfileSecrets, report: Report): await deleteObject(s, secrets, key) report('delete', 'ok', 'probe removed') } catch (e) { - report('delete', 'fail', `${errText(e)}. The probe object (a few bytes) stays at ${key}; delete it by hand.`, e instanceof S3Error && e.status === 0) + const cors = e instanceof S3Error && e.status === 0 && (await whyBlocked(apiUrl)) === 'cors' + report('delete', 'fail', `${errText(e)}. The probe object (a few bytes) stays at ${key}; delete it by hand.`, cors) + } +} + +/** Read `url` anonymously and require exactly `body`: a row's verdict. */ +async function readBack(url: string, body: Uint8Array, range: boolean): Promise<{ ok: true } | { ok: false; detail: string; cors: boolean }> { + try { + const r = await timedFetch(url, { ...(range ? { headers: { Range: `bytes=0-${body.length - 1}` } } : {}), credentials: 'omit', cache: 'no-store' }) + const got = await r.bytes() + if (!r.resp.ok) return { ok: false, detail: `HTTP ${r.resp.status}`, cors: false } + if (!bytesEqual(got, body)) return { ok: false, detail: 'served different bytes', cors: false } + return { ok: true } + } catch { + const why = await whyBlocked(url) + return { ok: false, detail: why === 'cors' ? 'refused this origin (CORS)' : why === 'redirect' ? 'redirected elsewhere (use 127.0.0.1, not localhost, for a local gateway)' : 'did not answer', cors: why === 'cors' } } } /** Run the IPFS test (kubo, plus the pinning service for a pinning profile). */ -async function probeIpfs(s: IpfsSettings, secrets: ProfileSecrets, report: Report): Promise { +async function probeIpfs(p: StorageProfile & { settings: IpfsSettings }, report: Report): Promise { + const s = p.settings + const secrets: ProfileSecrets = p.secrets report('api', 'running', '') try { report('api', 'ok', `kubo ${await kuboVersion(s, secrets)}`) } catch (e) { - const up = await reachable(s.api) - report('api', 'fail', up ? errText(e) : `${new URL(s.api).host} did not answer`, up) - for (const id of ['add', 'gateway', 'pinning', 'unpin'] as const) report(id, 'skipped', 'needs the kubo API') + const why = await whyBlocked(s.api) + report('api', 'fail', why === 'down' ? `${new URL(s.api).host} did not answer` : errText(e), why === 'cors') + for (const id of ['add', 'gateway', 'public-gateway', 'pinning', 'unpin'] as const) report(id, 'skipped', 'needs the kubo API') return } @@ -165,17 +219,20 @@ async function probeIpfs(s: IpfsSettings, secrets: ProfileSecrets, report: Repor report('gateway', 'running', '') if (cid === '' || s.gateway === '') report('gateway', 'skipped', s.gateway === '' ? 'no gateway configured: uploads are verified by CID and pin only' : 'needs the probe') else { - const url = gatewayUrl(s.gateway, cid) - try { - const resp = await fetch(url, { headers: { Range: `bytes=0-${body.length - 1}` }, credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(20_000) }) - const got = new Uint8Array(await resp.arrayBuffer()) - if (!resp.ok) report('gateway', 'fail', `HTTP ${resp.status} from the gateway`) - else if (!bytesEqual(got, body)) report('gateway', 'fail', 'the gateway served different bytes') - else report('gateway', 'ok', `re-read through ${new URL(url).host}`) - } catch { - const up = await reachable(url) - report('gateway', 'fail', up ? 'the gateway refused this origin (CORS)' : 'the gateway did not answer', up) - } + const r = await readBack(gatewayUrl(s.gateway, cid), body, true) + if (r.ok) report('gateway', 'ok', `re-read through ${new URL(s.gateway).host}`) + else report('gateway', 'fail', `the gateway ${r.detail}`, r.cors) + } + + report('public-gateway', 'running', '') + if (s.publicGateway === '') report('public-gateway', 'skipped', 'no public gateway: readers race public IPFS gateways, which may not find content only your node holds') + else if (cid === '') report('public-gateway', 'skipped', 'needs the probe') + else { + const unpublishable = publishProblem(p) + const r = await readBack(gatewayUrl(s.publicGateway, cid), body, false) + if (!r.ok) report('public-gateway', 'fail', `the public gateway ${r.detail}`, r.cors) + else if (unpublishable) report('public-gateway', 'fail', `Readable from here, but ${unpublishable}`) + else report('public-gateway', 'ok', `anonymous read through ${new URL(s.publicGateway).host}`) } report('pinning', 'running', '') @@ -213,7 +270,7 @@ export async function probeProfile(p: StorageProfile, report: Report): Promise seg === '' || seg === '.' || seg === '..')) return "the prefix has an empty, '.' or '..' segment" if (s.publicUrl === '') return 'a public URL is needed: browsers read packs anonymously from it' - const pu = checkUrl('the public URL', s.publicUrl) + const pu = checkUrl('the public URL', s.publicUrl, 'published') if (pu) return pu if (!p.secrets.accessKeyId || !p.secrets.secretAccessKey) return 'an access key id and secret are needed to sign uploads' return null } - const api = checkUrl('the kubo API', s.api, { originOnly: true }) + const api = checkUrl('the kubo API', s.api, 'local', { originOnly: true }) if (api) return api if (s.gateway !== '') { - const gw = checkUrl('the gateway', s.gateway, { originOnly: true }) + const gw = checkUrl('the gateway', s.gateway, 'local', { originOnly: true }) if (gw) return gw + if (new URL(s.gateway).hostname === 'localhost') { + return 'use http://127.0.0.1: for a local gateway: kubo redirects localhost to .ipfs.localhost subdomains, which the browser blocks' + } } if (s.publicGateway !== '') { - const pg = checkUrl('the public gateway', s.publicGateway, { originOnly: true }) + const pg = checkUrl('the public gateway', s.publicGateway, 'published', { originOnly: true }) if (pg) return pg } if (s.kind === 'ipfs-pinning-service') { - const pe = checkUrl('the pinning endpoint', s.pinningEndpoint) + const pe = checkUrl('the pinning endpoint', s.pinningEndpoint, 'local') if (pe) return pe if (!p.secrets.pinningToken) return 'the pinning service needs an access token' } return null } +const BUCKET = /^[a-z0-9._-]{1,63}$/ + +/** + * Why this profile's published addresses (the S3 public URL, the IPFS public gateway) cannot + * be recorded on chain, or null. They are what every reader fetches, so they must be public + * https: an address on this machine or a private network works only for its uploader, and would + * point other readers' browsers at their own local services. + */ +export function publishProblem(p: StorageProfile): string | null { + const s = p.settings + const published = s.kind === 's3' ? s.publicUrl : s.kind === 'platform' ? '' : s.publicGateway + if (published === '' || isPublicHttpsUrl(published)) return null + let host = published + try { + host = new URL(published).host + } catch { + /* keep the raw value */ + } + return `${host} is only reachable from this machine or its network, so other people cannot read what is stored there. The public address is recorded on chain for everyone: use a public https URL (a bucket domain, a CDN, or a tunnel).` +} + /** The key prefix normalized: '' or ending in exactly one `/` (parity with forge-core). */ export function normalizedPrefix(prefix: string): string { const p = prefix.replace(/^\/+|\/+$/g, '') @@ -254,35 +319,45 @@ export function artifactKey(settings: Extract, sh const httpString = z.string().max(300) -export const profilePublicSchema: z.ZodType = z.discriminatedUnion('kind', [ - z.object({ +const s3Schema = z + .object({ kind: z.literal('s3'), provider: z.enum(['r2', 'b2', 'aws', 'minio']), endpoint: httpString, - region: z.string().max(64), - bucket: z.string().max(63), + region: z.string().regex(/^[a-z0-9-]{1,64}$/), + bucket: z.string().regex(BUCKET), pathStyle: z.boolean(), publicUrl: httpString, prefix: z.string().max(200), - }), - z.object({ + }) + .strict() + +const ipfsSchema = z + .object({ kind: z.enum(['ipfs-kubo', 'ipfs-pinning-service']), provider: z.enum(['kubo', 'pinning']), api: httpString, gateway: httpString, publicGateway: httpString, pinningEndpoint: httpString, - }), - z.object({ kind: z.literal('platform'), provider: z.literal('platform') }), -]) as z.ZodType + }) + .strict() + // The kind and the tile must agree: a kubo tile is never a pinning profile, and back. + .refine((v) => (v.kind === 'ipfs-kubo') === (v.provider === 'kubo'), 'the IPFS kind and provider disagree') + +const platformSchema = z.object({ kind: z.literal('platform'), provider: z.literal('platform') }).strict() -export const profileSecretsSchema: z.ZodType = z +export const profilePublicSchema = z.union([s3Schema, ipfsSchema, platformSchema]) + +const headerValue = (max: number) => z.string().max(max).refine(isHeaderSafe).optional() + +export const profileSecretsSchema = z .object({ - accessKeyId: z.string().max(256).optional(), + accessKeyId: headerValue(256), secretAccessKey: z.string().max(512).optional(), - sessionToken: z.string().max(4096).optional(), - apiAuth: z.string().max(1024).optional(), - pinningToken: z.string().max(4096).optional(), + sessionToken: headerValue(4096), + apiAuth: headerValue(1024), + pinningToken: headerValue(4096), }) .strict() @@ -296,7 +371,7 @@ export interface StoragePolicy { readonly platformFallback: boolean } -export const policySchema: z.ZodType = z.object({ +export const policySchema = z.object({ targets: z.array(z.string().max(64)).max(8), replicas: z.number().int().min(1).max(8), platformFallback: z.boolean(), @@ -314,6 +389,11 @@ export function policyFor(targets: readonly string[], choice: ReplicationChoice) } } +/** `policy` with the profile `from` renamed to `to` (a renamed profile keeps its place). */ +export function renameInPolicy(policy: StoragePolicy, from: string, to: string): StoragePolicy { + return { ...policy, targets: policy.targets.map((t) => (t === from ? to : t)) } +} + /** Which replication choice a policy corresponds to (for the radio). */ export function choiceOf(policy: StoragePolicy): ReplicationChoice { if (policy.platformFallback) return 'fallback' diff --git a/forge-web/lib/storage/s3.ts b/forge-web/lib/storage/s3.ts index e33e03849..3f8a8eea8 100644 --- a/forge-web/lib/storage/s3.ts +++ b/forge-web/lib/storage/s3.ts @@ -12,12 +12,10 @@ import { EMPTY_PAYLOAD_SHA256, amzDate, keyHasBadSegment, sha256Hex, signRequest, uriEncode } from './sigv4' import type { ProfilePublic, ProfileSecrets } from './profiles' +import { TimeoutError, isHeaderSafe, timedFetch, type TimedResponse } from './util' export type S3Settings = Extract -/** How long one S3 request may take before it is abandoned. */ -export const S3_TIMEOUT_MS = 60_000 - /** An S3 request that failed: the operation, the HTTP status (0: no response) and why. */ export class S3Error extends Error { constructor( @@ -60,15 +58,18 @@ async function send( op: string, method: 'GET' | 'PUT' | 'HEAD' | 'DELETE', key: string, - opts: { body?: Uint8Array; contentType?: string; range?: string; signal?: AbortSignal } = {}, -): Promise { + opts: { body?: Uint8Array; contentType?: string; range?: string; signal?: AbortSignal | undefined } = {}, +): Promise { const url = objectUrl(s, key) const payloadHash = opts.body ? await sha256Hex(opts.body) : EMPTY_PAYLOAD_SHA256 const extra: [string, string][] = [] if (opts.contentType) extra.push(['content-type', opts.contentType]) if (opts.range) extra.push(['range', opts.range]) - const headers = new Headers(extra) if (!secrets.accessKeyId || !secrets.secretAccessKey) throw new S3Error(op, 0, 'no credentials: add the access key id and secret') + // A header value fetch refuses would be quoted in its error: refuse it here, without echoing it. + if (![secrets.accessKeyId, secrets.sessionToken ?? ''].every(isHeaderSafe)) { + throw new S3Error(op, 0, 'the access key id or session token holds characters an HTTP header cannot carry; paste it again') + } const signed = await signRequest( { method, @@ -87,30 +88,33 @@ async function send( }, amzDate(), ) - for (const [k, v] of signed) headers.set(k, v) - const timeout = AbortSignal.timeout(S3_TIMEOUT_MS) - const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout + const headers = new Headers([...extra, ...signed]) try { - return await fetch(url, { - method, - headers, - ...(opts.body ? { body: new Uint8Array(opts.body) } : {}), - redirect: 'error', - credentials: 'omit', - cache: 'no-store', - signal, - }) + return await timedFetch( + url, + { + method, + headers, + ...(opts.body ? { body: new Uint8Array(opts.body) } : {}), + redirect: 'error', + credentials: 'omit', + cache: 'no-store', + }, + { uploadBytes: opts.body?.length ?? 0, signal: opts.signal }, + ) } catch (e) { - // A CORS refusal and a network failure look the same to a page: say which is likely. - const why = timeout.aborted ? `no answer within ${S3_TIMEOUT_MS / 1000}s` : 'the request was blocked or the host is unreachable (a CORS rule missing for this origin, or the endpoint is down)' - throw new S3Error(op, 0, `${op} ${url.host}: ${why}${e instanceof Error && !timeout.aborted ? ` (${e.message})` : ''}`) + if (e instanceof TimeoutError) throw new S3Error(op, 0, `${op} ${url.host}: ${e.message}`) + // A CORS refusal, a redirect (refused: a signed request is never replayed elsewhere) and a + // network failure all look the same to a page; the probe tells them apart. + throw new S3Error(op, 0, `${op} ${url.host}: the request was blocked, redirected or the host is unreachable`) } } -async function failed(op: string, resp: Response): Promise { +async function failed(op: string, r: TimedResponse): Promise { + const resp = r.resp let code = '' try { - const text = (await resp.text()).slice(0, 2000) + const text = (await r.text()).slice(0, 2000) code = /([^<]{1,64})<\/Code>/.exec(text)?.[1] ?? '' } catch { /* body unreadable */ @@ -133,30 +137,40 @@ function statusHint(status: number): string { } } +/** + * An anonymous GET of `key` through the bucket's public URL (what every reader uses), whole or + * a `Range`: the check that a copy is readable by anyone, not just by the credential holder. + */ +export async function getPublic(s: S3Settings, key: string, range?: string): Promise<{ status: number; bytes: Uint8Array; contentRange: string | null }> { + const r = await timedFetch(publicObjectUrl(s, key), { credentials: 'omit', cache: 'no-store', ...(range ? { headers: { Range: range } } : {}) }) + return { status: r.resp.status, contentRange: r.resp.headers.get('content-range'), bytes: await r.bytes() } +} + /** Signed PUT of `bytes` at `key`. */ export async function putObject(s: S3Settings, secrets: ProfileSecrets, key: string, bytes: Uint8Array, contentType = 'application/octet-stream', signal?: AbortSignal): Promise { - const resp = await send(s, secrets, 'signed PUT', 'PUT', key, { body: bytes, contentType, signal }) - if (!resp.ok) throw await failed('signed PUT', resp) + const r = await send(s, secrets, 'signed PUT', 'PUT', key, { body: bytes, contentType, signal }) + if (!r.resp.ok) throw await failed('signed PUT', r) + await r.bytes() } /** Signed GET of `key` (whole object, or a `Range`). */ export async function getObject(s: S3Settings, secrets: ProfileSecrets, key: string, range?: string, signal?: AbortSignal): Promise { - const resp = await send(s, secrets, 'signed GET', 'GET', key, { range, signal }) - if (!resp.ok) throw await failed('signed GET', resp) - return new Uint8Array(await resp.arrayBuffer()) + const r = await send(s, secrets, 'signed GET', 'GET', key, { range, signal }) + if (!r.resp.ok) throw await failed('signed GET', r) + return r.bytes() } /** Signed HEAD: the object's size, or null when it does not exist. */ export async function headObject(s: S3Settings, secrets: ProfileSecrets, key: string): Promise { - const resp = await send(s, secrets, 'signed HEAD', 'HEAD', key) - if (resp.status === 404) return null - if (!resp.ok) throw await failed('signed HEAD', resp) - const len = Number(resp.headers.get('content-length')) + const r = await send(s, secrets, 'signed HEAD', 'HEAD', key) + if (r.resp.status === 404) return null + if (!r.resp.ok) throw await failed('signed HEAD', r) + const len = Number(r.resp.headers.get('content-length')) return Number.isFinite(len) ? len : 0 } /** Signed DELETE of `key` (idempotent: a missing object is fine). */ export async function deleteObject(s: S3Settings, secrets: ProfileSecrets, key: string): Promise { - const resp = await send(s, secrets, 'delete', 'DELETE', key) - if (!resp.ok && resp.status !== 404) throw await failed('delete', resp) + const r = await send(s, secrets, 'delete', 'DELETE', key) + if (!r.resp.ok && r.resp.status !== 404) throw await failed('delete', r) } diff --git a/forge-web/lib/storage/storage.test.ts b/forge-web/lib/storage/storage.test.ts index 91fb1a22a..9ff03d165 100644 --- a/forge-web/lib/storage/storage.test.ts +++ b/forge-web/lib/storage/storage.test.ts @@ -6,27 +6,34 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { resetMemoryStores, idbEntries } from '../idb' +import { resetMemoryStores, idbEntries, idbGet, idbPut } from '../idb' import { lockVault, storeInVault, unlockWithPassphrase } from '../auth/vault' import { splitChunks, manifestUrisProblem } from '../repo/push' import { CHUNK_PAYLOAD_MAX, FIELD_MAX } from '../constants' +import { LOCAL_HTTP_ALLOWED, isPrivateHost, isPublicHttpsUrl } from '../net' +import { externalFetchUrls } from '../view/browse-source' import { corsFix } from './cors' import { cidV1RawLeaves } from './cid' import { + PLATFORM_PROFILE, artifactKey, choiceOf, policyFor, profileProblem, + profileSecretsSchema, + publishProblem, type ProfilePublic, type StorageProfile, } from './profiles' import { EMPTY_STORAGE_CONFIG, + discardStorageConfig, loadStorageConfig, policyForRepo, policyProblem, saveStorageConfig, withProfile, + withRenamedProfile, withRepoPolicy, withoutProfile, } from './store' @@ -37,46 +44,74 @@ import type { V2RepoRef } from '../repo/contract' import type { EvoSDK } from '@dashevo/evo-sdk' const ID = '9r27eDsuXEqoMNymW1A2MKFrpBhzSkepVKwXrGzq9dUD' +const WIF = 'cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy' const S3: StorageProfile = { - name: 'minio', + name: 'r2-main', settings: { kind: 's3', - provider: 'minio', - endpoint: 'http://127.0.0.1:9000', - region: 'us-east-1', + provider: 'r2', + endpoint: 'https://acct.r2.cloudflarestorage.com', + region: 'auto', bucket: 'forge-byo', pathStyle: true, - publicUrl: 'http://127.0.0.1:9000/forge-byo', + publicUrl: 'https://pub-9a1.r2.dev', prefix: 'web', }, - secrets: { accessKeyId: 'minioadmin', secretAccessKey: 'minio-secret-value' }, + secrets: { accessKeyId: 'AKIDEXAMPLE', secretAccessKey: 'r2-secret-value' }, } const KUBO: StorageProfile = { name: 'kubo', - settings: { kind: 'ipfs-kubo', provider: 'kubo', api: 'http://127.0.0.1:5001', gateway: 'http://127.0.0.1:8081', publicGateway: '', pinningEndpoint: '' }, + settings: { kind: 'ipfs-kubo', provider: 'kubo', api: 'https://kubo.example', gateway: 'https://kubo-gw.example', publicGateway: '', pinningEndpoint: '' }, secrets: {}, } +const s3Of = (p: StorageProfile): Extract => p.settings as Extract + describe('profiles', () => { it('accepts a sound S3 profile and names what is wrong otherwise', () => { expect(profileProblem(S3)).toBeNull() - const s3 = S3.settings as Extract + const s3 = s3Of(S3) const bad = (patch: Partial, secrets = S3.secrets): string | null => profileProblem({ ...S3, settings: { ...s3, ...patch }, secrets }) expect(bad({ endpoint: 'https://x.example/bucket' })).toMatch(/origin/) expect(bad({ endpoint: 'https://user:pw@x.example' })).toMatch(/user name/) expect(bad({ endpoint: 'http://minio.example.org' })).toMatch(/https/) + // Plain http to this machine is a dev/devnet-build allowance only (as is the CSP's). + if (LOCAL_HTTP_ALLOWED) expect(bad({ endpoint: 'http://127.0.0.1:9000' })).toBeNull() + else expect(bad({ endpoint: 'http://127.0.0.1:9000' })).toMatch(/https/) expect(bad({ bucket: 'Forge' })).toMatch(/lowercase/) expect(bad({ prefix: 'a/../b' })).toMatch(/segment/) expect(bad({ publicUrl: '' })).toMatch(/public URL/) + expect(bad({ publicUrl: 'http://pub.example' })).toMatch(/recorded on chain/) expect(bad({ pathStyle: false, endpoint: 'https://1.2.3.4' })).toMatch(/path-style/) expect(bad({}, {})).toMatch(/access key/) expect(profileProblem({ ...S3, name: 'a,b' })).toMatch(/name/) }) + it('reserves the Platform profile name', () => { + expect(profileProblem({ ...S3, name: PLATFORM_PROFILE })).toMatch(/reserved/) + expect(profileProblem({ name: PLATFORM_PROFILE, settings: { kind: 'platform', provider: 'platform' }, secrets: {} })).toBeNull() + }) + + it('refuses secrets an HTTP header cannot carry, without echoing them', () => { + const problem = profileProblem({ ...S3, secrets: { ...S3.secrets, sessionToken: 'tok\nsecret-part' } }) + expect(problem).toMatch(/session token/i) + expect(problem).not.toContain('secret-part') + expect(profileSecretsSchema.safeParse({ accessKeyId: 'ключ' }).success).toBe(false) + expect(profileSecretsSchema.safeParse({ accessKeyId: 'AKID', extra: 'x' }).success).toBe(false) + }) + + it('refuses a published address only its uploader can reach', () => { + expect(publishProblem(S3)).toBeNull() + for (const url of ['http://127.0.0.1:9000/forge-byo', 'https://192.168.1.4/b', 'https://10.0.0.8', 'https://minio.local/b', 'https://[::1]/b', 'https://[fd00::1]/b']) { + expect(publishProblem({ ...S3, settings: { ...s3Of(S3), publicUrl: url } }), url).toMatch(/other people cannot read/) + } + expect(publishProblem({ ...KUBO, settings: { ...KUBO.settings, publicGateway: 'http://localhost:8080' } as ProfilePublic })).toMatch(/other people/) + }) + it('keys artifacts like the CLI', () => { - const s3 = S3.settings as Extract + const s3 = s3Of(S3) expect(artifactKey(s3, 'ab'.repeat(32))).toBe(`web/packs/${'ab'.repeat(32)}.pack`) expect(artifactKey({ ...s3, prefix: '' }, 'cd')).toBe('packs/cd.pack') expect(artifactKey({ ...s3, prefix: '/a/b//' }, 'cd')).toBe('a/b/packs/cd.pack') @@ -90,23 +125,51 @@ describe('profiles', () => { }) }) +describe('what readers may fetch', () => { + it('only public https hosts', () => { + for (const h of ['127.0.0.1', 'localhost', 'x.localhost', '10.1.2.3', '172.20.0.1', '192.168.0.9', '169.254.1.1', '100.64.0.1', '::1', 'fe80::1', 'fc00::5', '::ffff:127.0.0.1']) { + expect(isPrivateHost(h), h).toBe(true) + } + for (const h of ['pub-9a1.r2.dev', '8.8.8.8', '172.32.0.1', 'ipfs.io']) expect(isPrivateHost(h), h).toBe(false) + expect(isPublicHttpsUrl('http://pub.example/p')).toBe(false) + expect(isPublicHttpsUrl('https://pub.example/p')).toBe(true) + }) + + it('never makes a reader request a loopback or private URL a manifest records', () => { + expect( + externalFetchUrls(['http://127.0.0.1:9000/p', 'https://192.168.1.2/p', 'http://pub.example/p', 'https://pub.example/p', 'ipfs://bafyok', 'ipfs://bafy/../api'], ['https://ipfs.io']), + ).toEqual(['https://pub.example/p', 'https://ipfs.io/ipfs/bafyok']) + }) +}) + describe('the stored configuration', () => { it('prunes a removed profile out of every policy', () => { let c = withProfile(withProfile(EMPTY_STORAGE_CONFIG, S3), KUBO) - c = { ...c, defaultPolicy: policyFor(['minio', 'kubo'], 'all') } + c = { ...c, defaultPolicy: policyFor(['r2-main', 'kubo'], 'all') } c = withRepoPolicy(c, 'R1', policyFor(['kubo'], 'one')) const after = withoutProfile(c, 'kubo') - expect(after.profiles.map((p) => p.name)).toEqual(['minio']) - expect(after.defaultPolicy).toEqual({ targets: ['minio'], replicas: 1, platformFallback: false }) + expect(after.profiles.map((p) => p.name)).toEqual(['r2-main']) + expect(after.defaultPolicy).toEqual({ targets: ['r2-main'], replicas: 1, platformFallback: false }) expect(after.repoPolicies).toEqual({}) expect(policyForRepo(after, 'R1')).toEqual(after.defaultPolicy) }) + it('renames a profile inside every policy', () => { + let c = withProfile(withProfile(EMPTY_STORAGE_CONFIG, S3), KUBO) + c = { ...c, defaultPolicy: policyFor(['r2-main', 'kubo'], 'all'), lastTests: { kubo: { at: 1, ok: true } } } + c = withRepoPolicy(c, 'R1', policyFor(['kubo'], 'one')) + const after = withRenamedProfile(c, 'kubo', { ...KUBO, name: 'my-node' }) + expect(after.profiles.map((p) => p.name)).toEqual(['my-node', 'r2-main']) + expect(after.defaultPolicy?.targets).toEqual(['r2-main', 'my-node']) + expect(after.repoPolicies['R1']?.targets).toEqual(['my-node']) + expect(after.lastTests).toEqual({}) + }) + it('refuses a policy naming unknown profiles or too many copies', () => { const c = withProfile(EMPTY_STORAGE_CONFIG, S3) - expect(policyProblem(c, policyFor(['minio'], 'one'))).toBeNull() + expect(policyProblem(c, policyFor(['r2-main'], 'one'))).toBeNull() expect(policyProblem(c, policyFor(['nope'], 'one'))).toMatch(/no storage profile/) - expect(policyProblem(c, { targets: ['minio'], replicas: 2, platformFallback: false })).toMatch(/copies/) + expect(policyProblem(c, { targets: ['r2-main'], replicas: 2, platformFallback: false })).toMatch(/copies/) expect(policyProblem(c, { targets: [], replicas: 1, platformFallback: false })).toMatch(/at least one/) }) }) @@ -118,27 +181,47 @@ describe('the configuration sealed in the vault', () => { }) it('stores secrets encrypted, reads them back unlocked, and refuses when locked', async () => { - await storeInVault('devnet', { identityId: ID, keyId: 5, wif: 'cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy' }, { passphrase: 'correct horse battery' }) - const config = withRepoPolicy(withProfile(EMPTY_STORAGE_CONFIG, S3), 'R1', policyFor(['minio'], 'one')) + await storeInVault('devnet', { identityId: ID, keyId: 5, wif: WIF }, { passphrase: 'correct horse battery' }) + const config = withRepoPolicy(withProfile(EMPTY_STORAGE_CONFIG, S3), 'R1', policyFor(['r2-main'], 'one')) await saveStorageConfig('devnet', ID, config) const dump = JSON.stringify(await idbEntries('vault'), (_k, v: unknown) => (v instanceof Uint8Array ? Array.from(v) : v)) - expect(dump).not.toContain('minio-secret-value') + expect(dump).not.toContain('r2-secret-value') expect(dump).not.toContain('forge-byo') expect(await loadStorageConfig('devnet', ID)).toEqual(config) lockVault() await expect(loadStorageConfig('devnet', ID)).rejects.toThrow(/unlock/) await unlockWithPassphrase('devnet', ID, 'correct horse battery') - expect((await loadStorageConfig('devnet', ID)).profiles[0]?.secrets.secretAccessKey).toBe('minio-secret-value') + expect((await loadStorageConfig('devnet', ID)).profiles[0]?.secrets.secretAccessKey).toBe('r2-secret-value') }, 60_000) - it('carries the storage settings across a key renewal', async () => { - const wif = 'cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy' - await storeInVault('devnet', { identityId: ID, keyId: 5, wif }, { passphrase: 'correct horse battery' }) + it('carries the storage settings across a key renewal made while unlocked', async () => { + await storeInVault('devnet', { identityId: ID, keyId: 5, wif: WIF }, { passphrase: 'correct horse battery' }) await saveStorageConfig('devnet', ID, withProfile(EMPTY_STORAGE_CONFIG, S3)) - await storeInVault('devnet', { identityId: ID, keyId: 6, wif }, { passphrase: 'a different passphrase' }) + const outcome = await storeInVault('devnet', { identityId: ID, keyId: 6, wif: WIF }, { passphrase: 'a different passphrase' }) + expect(outcome.storageSettingsDropped).toBe(false) lockVault() await unlockWithPassphrase('devnet', ID, 'a different passphrase') - expect((await loadStorageConfig('devnet', ID)).profiles.map((p) => p.name)).toEqual(['minio']) + expect((await loadStorageConfig('devnet', ID)).profiles.map((p) => p.name)).toEqual(['r2-main']) + }, 90_000) + + it('says so when a renewal made while locked cannot carry them', async () => { + await storeInVault('devnet', { identityId: ID, keyId: 5, wif: WIF }, { passphrase: 'correct horse battery' }) + await saveStorageConfig('devnet', ID, withProfile(EMPTY_STORAGE_CONFIG, S3)) + lockVault() + const outcome = await storeInVault('devnet', { identityId: ID, keyId: 6, wif: WIF }, { passphrase: 'a different passphrase' }) + expect(outcome.storageSettingsDropped).toBe(true) + expect(await loadStorageConfig('devnet', ID)).toEqual(EMPTY_STORAGE_CONFIG) + }, 90_000) + + it('can discard settings sealed under a key it no longer has', async () => { + await storeInVault('devnet', { identityId: ID, keyId: 5, wif: WIF }, { passphrase: 'correct horse battery' }) + await saveStorageConfig('devnet', ID, withProfile(EMPTY_STORAGE_CONFIG, S3)) + const stale = await idbGet('vault', `vault-storage:devnet:${ID}`) + await storeInVault('devnet', { identityId: ID, keyId: 6, wif: WIF }, { passphrase: 'a different passphrase' }) + await idbPut('vault', `vault-storage:devnet:${ID}`, stale) + await expect(loadStorageConfig('devnet', ID)).rejects.toThrow(/do not open/) + await discardStorageConfig('devnet', ID) + expect(await loadStorageConfig('devnet', ID)).toEqual(EMPTY_STORAGE_CONFIG) }, 90_000) }) @@ -153,7 +236,7 @@ describe('chunks and manifests', () => { expect(joined).toEqual(bytes) }) - it('orders URIs platform, then http(s), then the rest, and fits the budget', () => { + it('orders URIs platform, then https, then the rest, and fits the budget', () => { expect(orderUris([['https://a/p', 's3://b/p'], ['ipfs://bafy', 'https://gw/ipfs/bafy'], ['platform://C/R/O/h']])).toEqual([ 'platform://C/R/O/h', 'https://a/p', @@ -161,26 +244,43 @@ describe('chunks and manifests', () => { 's3://b/p', 'ipfs://bafy', ]) - const many = Array.from({ length: 5 }, (_, i) => [`https://h${i}/p`, `s3://b${i}/p`]) + const many = Array.from({ length: 5 }, (_, i) => [`https://h${i}.example/p`, `s3://b${i}/p`]) expect(fitManifestUris(orderUris(many))).toHaveLength(5) - expect(() => fitManifestUris([`https://x/${'a'.repeat(400)}`])).toThrow(/do not fit/) + expect(() => fitManifestUris([`https://x.example/${'a'.repeat(400)}`])).toThrow(/do not fit/) expect(manifestUrisProblem([])).toMatch(/no confirmed copy/) }) + + it('never records an address others cannot read', () => { + expect(() => fitManifestUris(['http://127.0.0.1:9000/b/p', 's3://b/p'])).toThrow(/only public https/) + expect(() => fitManifestUris(['https://192.168.1.2/p'])).toThrow(/only public https/) + expect(() => fitManifestUris(['http://pub.example/p'])).toThrow(/only public https/) + }) }) describe('CORS fix blocks', () => { it('are valid JSON naming the bucket, Range for reads and PUT for this origin', () => { for (const p of ['r2', 'aws', 'b2'] as const) { const fix = corsFix(p, 'my-bucket', 'https://forge.dashhq.org') - expect(fix.where + fix.text).toContain('my-bucket'.slice(0, p === 'r2' || p === 'aws' || p === 'b2' ? 9 : 0)) - const json: unknown = JSON.parse(fix.text) - const text = JSON.stringify(json).toLowerCase() + expect(fix.where + fix.text).toContain('my-bucket') + const text = JSON.stringify(JSON.parse(fix.text) as unknown).toLowerCase() expect(text).toContain('range') expect(text).toMatch(/put/) expect(text).toContain('https://forge.dashhq.org') expect(text).toContain('x-amz-content-sha256') } - expect(corsFix('kubo', '', 'https://forge.dashhq.org').text).toContain('API.HTTPHeaders.Access-Control-Allow-Origin') + }) + + it('keeps MinIO reads open to every origin', () => { + expect(corsFix('minio', 'b', 'https://forge.dashhq.org').text).toContain('cors_allow_origin="*"') + }) + + it('gives kubo a restricted token and merges the origin into the existing list', () => { + const { where, text } = corsFix('kubo', '', 'https://forge.dashhq.org') + expect(where).toMatch(/admin interface/) + expect(text).toContain('API.Authorizations.dash-forge') + for (const path of ['/api/v0/add', '/api/v0/pin/ls', '/api/v0/pin/rm', '/api/v0/id', '/api/v0/version']) expect(text).toContain(path) + expect(text).toMatch(/\(\. \/\/ \[\]\) \+ \["https:\/\/forge\.dashhq\.org"\] \| unique/) + expect(text).not.toMatch(/Access-Control-Allow-Origin '\["https:\/\/forge\.dashhq\.org"\]'/) }) }) @@ -192,16 +292,25 @@ const REPO: V2RepoRef = { kind: 'v2', forge: { core: 'CORE', collab: 'COLLAB', g const AUTH: WriteAuth = { identityId: ID, network: 'devnet', getSigningKeyWif: () => '' } const SDK = {} as EvoSDK -function fakeNetwork(opts: { s3Down?: boolean; corruptOnce?: boolean } = {}) { +/** Serve `obj` for a (possibly ranged) GET. */ +function serve(obj: Uint8Array, init?: RequestInit): Response { + const range = new Headers(init?.headers).get('range') + const m = range ? /^bytes=(\d+)-(\d+)$/.exec(range) : null + if (!m) return new Response(new Uint8Array(obj), { status: 200 }) + const [start, end] = [Number(m[1]), Number(m[2])] + return new Response(obj.slice(start, end + 1), { status: 206, headers: { 'content-range': `bytes ${start}-${end}/${obj.length}` } }) +} + +function fakeNetwork(opts: { s3Down?: boolean; corruptOnce?: boolean; publicDenied?: boolean } = {}) { const objects = new Map() let corrupt = opts.corruptOnce === true const puts: string[] = [] const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit): Promise => { const url = new URL(String(input)) const method = init?.method ?? 'GET' - if (url.port === '9000') { + if (url.host === 'acct.r2.cloudflarestorage.com') { if (opts.s3Down) throw new TypeError('Failed to fetch') - const key = url.pathname + const key = url.pathname.replace(/^\/forge-byo\//, '') if (method === 'PUT') { const body = new Uint8Array(init?.body as ArrayBuffer | Uint8Array) puts.push(key) @@ -212,9 +321,14 @@ function fakeNetwork(opts: { s3Down?: boolean; corruptOnce?: boolean } = {}) { const obj = objects.get(key) if (!obj) return new Response(null, { status: 404 }) if (method === 'HEAD') return new Response(null, { status: 200, headers: { 'content-length': String(obj.length) } }) - return new Response(new Uint8Array(obj), { status: 200 }) + return serve(obj, init) + } + if (url.host === 'pub-9a1.r2.dev') { + if (opts.publicDenied) return new Response('AccessDenied', { status: 403 }) + const obj = objects.get(url.pathname.slice(1)) + return obj ? serve(obj, init) : new Response(null, { status: 404 }) } - if (url.port === '5001') { + if (url.host === 'kubo.example') { if (url.pathname.endsWith('/add')) { const file = (init?.body as FormData).get('file') as Blob const bytes = new Uint8Array(await file.arrayBuffer()) @@ -228,9 +342,9 @@ function fakeNetwork(opts: { s3Down?: boolean; corruptOnce?: boolean } = {}) { } return new Response('{}') } - if (url.port === '8081') { + if (url.host === 'kubo-gw.example' || url.host === 'ipfs.example') { const obj = objects.get(`ipfs:${url.pathname.split('/').pop() ?? ''}`) - return obj ? new Response(new Uint8Array(obj)) : new Response(null, { status: 404 }) + return obj ? serve(obj, init) : new Response(null, { status: 404 }) } throw new Error(`unexpected ${url}`) }) @@ -241,44 +355,70 @@ describe('storeArtifact', () => { const bytes = new TextEncoder().encode('PACK a small pack') afterEach(() => vi.unstubAllGlobals()) - it('stores on S3, verifies by re-read, and records the public URL before the s3:// locator', async () => { + it('stores on S3, verifies through the API and the public URL, and records the public URL first', async () => { const net = fakeNetwork() vi.stubGlobal('fetch', net.fetchMock) - const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { - policy: policyFor(['minio'], 'one'), - profiles: [S3], - confirmPlatform: async () => false, - }) + const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main'], 'one'), profiles: [S3], confirmPlatform: async () => false }) const hash = await sha256Hex(bytes) - expect(stored).toMatchObject({ packHash: hash, storage: 1, chunkCount: 0, confirmed: ['minio'], failures: [] }) - expect(stored.uris).toEqual([`http://127.0.0.1:9000/forge-byo/web/packs/${hash}.pack`, `s3://forge-byo/web/packs/${hash}.pack`]) - // The request that wrote it was signed. + expect(stored).toMatchObject({ packHash: hash, storage: 1, chunkCount: 0, confirmed: ['r2-main'], failures: [] }) + expect(stored.uris).toEqual([`https://pub-9a1.r2.dev/web/packs/${hash}.pack`, `s3://forge-byo/web/packs/${hash}.pack`]) const put = net.fetchMock.mock.calls.find(([, init]) => init?.method === 'PUT') - expect(new Headers(put?.[1]?.headers).get('authorization')).toMatch(/^AWS4-HMAC-SHA256 Credential=minioadmin\//) + expect(new Headers(put?.[1]?.headers).get('authorization')).toMatch(/^AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE\//) expect(new Headers(put?.[1]?.headers).get('x-amz-content-sha256')).toBe(hash) + // The public URL was read anonymously (no Authorization) before the copy counted. + const pub = net.fetchMock.mock.calls.find(([u]) => String(u).startsWith('https://pub-9a1.r2.dev/')) + expect(pub).toBeDefined() + expect(new Headers(pub?.[1]?.headers).get('authorization')).toBeNull() + }) + + it('does not count a copy whose public URL others cannot read', async () => { + vi.stubGlobal('fetch', fakeNetwork({ publicDenied: true }).fetchMock) + await expect(storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main'], 'one'), profiles: [S3], confirmPlatform: async () => false })).rejects.toThrow( + /public URL \(pub-9a1\.r2\.dev\) answered HTTP 403/, + ) + }) + + it('refuses to upload to a profile whose public address only this machine can read', async () => { + const net = fakeNetwork() + vi.stubGlobal('fetch', net.fetchMock) + const local = { ...S3, settings: { ...s3Of(S3), publicUrl: 'https://192.168.1.20/forge-byo' } } + await expect(storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main'], 'one'), profiles: [local], confirmPlatform: async () => false })).rejects.toThrow(/other people cannot read/) + expect(net.puts).toEqual([]) }) it('re-uploads once when the stored copy does not verify', async () => { const net = fakeNetwork({ corruptOnce: true }) vi.stubGlobal('fetch', net.fetchMock) - const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['minio'], 'one'), profiles: [S3], confirmPlatform: async () => false }) - expect(stored.confirmed).toEqual(['minio']) + const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main'], 'one'), profiles: [S3], confirmPlatform: async () => false }) + expect(stored.confirmed).toEqual(['r2-main']) expect(net.puts).toHaveLength(2) }) it('fails the policy (writing nothing to Platform) when too few targets confirm', async () => { vi.stubGlobal('fetch', fakeNetwork({ s3Down: true }).fetchMock) const confirm = vi.fn(async () => true) - const run = storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['minio', 'kubo'], 'all'), profiles: [S3, KUBO], confirmPlatform: confirm }) + const run = storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main', 'kubo'], 'all'), profiles: [S3, KUBO], confirmPlatform: confirm }) await expect(run).rejects.toBeInstanceOf(ReplicationError) - await expect(run).rejects.toThrow(/1 of 2 required .*minio: signed PUT.*Nothing was written to Platform/) + await expect(run).rejects.toThrow(/1 of 2 required .*r2-main: signed PUT.*Nothing was written to Platform/) expect(confirm).not.toHaveBeenCalled() }) - it('verifies the kubo CID against the local derivation', async () => { + it('asks, with the price, before a policy that names Platform writes any chunk', async () => { + vi.stubGlobal('fetch', fakeNetwork().fetchMock) + const platform: StorageProfile = { name: PLATFORM_PROFILE, settings: { kind: 'platform', provider: 'platform' }, secrets: {} } + const confirm = vi.fn(async () => false) + const run = storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor([PLATFORM_PROFILE], 'one'), profiles: [platform], confirmPlatform: confirm }) + await expect(run).rejects.toBeInstanceOf(ReplicationError) + await expect(run).rejects.toThrow(/declined.*Nothing was written to Platform/) + expect(confirm).toHaveBeenCalledWith(expect.objectContaining({ bytes: bytes.length, estimateCredits: expect.any(Number), reason: expect.stringMatching(/includes Dash Platform/) })) + }) + + it('verifies the kubo CID, the node gateway and the public gateway', async () => { vi.stubGlobal('fetch', fakeNetwork().fetchMock) - const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['kubo'], 'one'), profiles: [{ ...KUBO, settings: { ...KUBO.settings, gateway: '' } as ProfilePublic }], confirmPlatform: async () => false }) - expect(stored.uris).toEqual([`ipfs://${cidV1RawLeaves(bytes)}`]) + const withPublic = { ...KUBO, settings: { ...KUBO.settings, publicGateway: 'https://ipfs.example' } as ProfilePublic } + const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['kubo'], 'one'), profiles: [withPublic], confirmPlatform: async () => false }) + const cid = cidV1RawLeaves(bytes) + expect(stored.uris).toEqual([`https://ipfs.example/ipfs/${cid}`, `ipfs://${cid}`]) }) it('asks before storing on Platform when nothing is configured, and stops when declined', async () => { diff --git a/forge-web/lib/storage/store.ts b/forge-web/lib/storage/store.ts index 96734fd2a..bdce94349 100644 --- a/forge-web/lib/storage/store.ts +++ b/forge-web/lib/storage/store.ts @@ -14,9 +14,10 @@ import { z } from 'zod' import type { Network } from '../constants' -import { readStorageBlob, writeStorageBlob } from '../auth/vault' +import { discardStorageBlob, readStorageBlob, writeStorageBlob } from '../auth/vault' import { policySchema, + renameInPolicy, profilePublicSchema, profileSecretsSchema, type StoragePolicy, @@ -68,6 +69,36 @@ export async function saveStorageConfig(network: Network, identityId: string, co await writeStorageBlob(network, identityId, parsed) } +/** + * Delete stored storage settings this key cannot open (sealed under an earlier key, or in a + * shape this app does not understand). The profiles in them are lost: add them again. + */ +export function discardStorageConfig(network: Network, identityId: string): Promise { + return discardStorageBlob(network, identityId) +} + +/** + * `config` with the profile `from` replaced by `profile`: every policy that named it follows + * a rename, as does its last test result. + */ +export function withRenamedProfile(config: StorageConfig, from: string, profile: StorageProfile): StorageConfig { + const to = profile.name + const repoPolicies: Record = {} + for (const [repo, p] of Object.entries(config.repoPolicies)) repoPolicies[repo] = renameInPolicy(p, from, to) + const lastTests = { ...config.lastTests } + delete lastTests[from] + return withProfile( + { + ...config, + profiles: config.profiles.filter((p) => p.name !== from), + defaultPolicy: config.defaultPolicy ? renameInPolicy(config.defaultPolicy, from, to) : null, + repoPolicies, + lastTests, + }, + profile, + ) +} + /** `config` with `profile` added or replaced (by name). */ export function withProfile(config: StorageConfig, profile: StorageProfile): StorageConfig { const profiles = [...config.profiles.filter((p) => p.name !== profile.name), profile].sort((a, b) => a.name.localeCompare(b.name)) diff --git a/forge-web/lib/storage/upload.live.test.ts b/forge-web/lib/storage/upload.live.test.ts index f3b95d5fd..8025128e2 100644 --- a/forge-web/lib/storage/upload.live.test.ts +++ b/forge-web/lib/storage/upload.live.test.ts @@ -2,7 +2,7 @@ * Live browser-upload path — SKIPPED by default (devnet writes + a local MinIO). * * docker compose -f infra/docker-compose.yml up -d minio minio-init - * FORGE_LIVE=1 NEXT_PUBLIC_NETWORK=devnet NEXT_PUBLIC_DEVNET_NAME=moutai \ + * FORGE_LIVE=1 FORGE_LIVE_PUBLIC_URL=https://…/forge-byo NEXT_PUBLIC_NETWORK=devnet NEXT_PUBLIC_DEVNET_NAME=moutai \ * pnpm exec vitest run lib/storage/upload.live.test.ts * * As the moutai MAINTAINER (a maintainer of its own `forge-v2-empty`, the fixture no push has @@ -21,6 +21,7 @@ import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { DEFAULT_NETWORK, NETWORKS } from '../constants' +import { isPublicHttpsUrl } from '../net' import { evoSdkService, type WriteAuth } from '../sdk' import { parseIdentityFileText } from '../auth' import { readV2PackCopies } from '../repo' @@ -30,7 +31,13 @@ import type { V2RepoRef } from '../repo/contract' import { storeAndRecordPack, policyFor, type StorageProfile } from './index' import { sha256Hex } from './sigv4' -const LIVE = process.env['FORGE_LIVE'] === '1' && DEFAULT_NETWORK === 'devnet' +/** + * The bucket's PUBLIC https address: a manifest records only addresses anyone can read, so a + * bare local MinIO does not qualify. Expose it for the run, e.g. + * `cloudflared tunnel --url http://127.0.0.1:9000` → FORGE_LIVE_PUBLIC_URL=https://.trycloudflare.com/forge-byo + */ +const PUBLIC_URL = (process.env['FORGE_LIVE_PUBLIC_URL'] ?? '').replace(/\/+$/, '') +const LIVE = process.env['FORGE_LIVE'] === '1' && DEFAULT_NETWORK === 'devnet' && isPublicHttpsUrl(PUBLIC_URL) const MAINTAINER_FILE = join(homedir(), '.config/dash-forge/test-identities/devnet-moutai/MAINTAINER.identity.json') const EMPTY_REPO = 'F9puk5NBQyySFbgj9yUfubdXAVv1YV7Zk2KDvfyHzxLU' @@ -43,7 +50,7 @@ const MINIO: StorageProfile = { region: 'us-east-1', bucket: 'forge-byo', pathStyle: true, - publicUrl: 'http://127.0.0.1:9000/forge-byo', + publicUrl: PUBLIC_URL, prefix: 'web-live', }, secrets: { accessKeyId: 'minioadmin', secretAccessKey: 'minioadmin' }, @@ -61,7 +68,9 @@ describe.skipIf(!LIVE)('live browser upload to MinIO + manifest on moutai', () = const sdk = evoSdkService.getSdk() const repo: V2RepoRef = { kind: 'v2', forge: v2, repoId: EMPTY_REPO, ownerId: parsed.identityId, name: 'forge-v2-empty', visibility: 'public' } - const bytes = new TextEncoder().encode(`forge-web live upload artifact ${EMPTY_REPO}\n`) + // One artifact per public URL: the signer's manifest slot for a pack is permanent, so a + // rerun with the same bytes and URL finds its own manifest and writes nothing. + const bytes = new TextEncoder().encode(`forge-web live upload artifact ${EMPTY_REPO} ${PUBLIC_URL}\n`) const hash = await sha256Hex(bytes) const events: string[] = [] const { stored, manifest } = await storeAndRecordPack( @@ -81,7 +90,7 @@ describe.skipIf(!LIVE)('live browser upload to MinIO + manifest on moutai', () = // eslint-disable-next-line no-console console.log('stored', { uris: stored.uris, manifest: manifest.documentId, cost: manifest.cost.dash, events }) expect(stored.storage).toBe(1) - expect(stored.uris[0]).toBe(`http://127.0.0.1:9000/forge-byo/web-live/packs/${hash}.pack`) + expect(stored.uris[0]).toBe(`${PUBLIC_URL}/web-live/packs/${hash}.pack`) expect(stored.uris[1]).toBe(`s3://forge-byo/web-live/packs/${hash}.pack`) expect(events).toEqual(['minio:start', 'minio:done']) diff --git a/forge-web/lib/storage/upload.ts b/forge-web/lib/storage/upload.ts index b1fbfb043..77e4f8a33 100644 --- a/forge-web/lib/storage/upload.ts +++ b/forge-web/lib/storage/upload.ts @@ -4,27 +4,34 @@ * * Order of events, the same guarantee the CLI gives: the bytes are stored AND verified on at * least `replicas` targets before this resolves, and only then may the caller write the - * `packManifest` (and after it, refs). Nothing on Platform is written here unless Platform is - * itself the target (or the fallback the user confirmed, with its price). + * `packManifest` (and after it, refs). Nothing is written to Platform without the user first + * agreeing to its price ({@link StoreOptions.confirmPlatform}) — asked once, before the first + * chunk, whether Platform is a target of the policy, the fallback, or the only place. * - * Verification is by re-read, as in the CLI: S3 — HEAD, then the whole object up to 16 MiB - * re-hashed (larger: size plus byte-exact head and tail windows); IPFS — kubo's CID must equal - * the local derivation and be pinned, then a gateway re-read when one is configured. - * S3 object keys are content-addressed (`packs/.pack`), so a re-upload is - * idempotent and a same-size object at the key is re-verified rather than trusted. + * A copy counts as confirmed only when the URLs it records are readable by anyone, as in the + * CLI (`ExternalTarget::store` re-reads through the first recorded URI): + * - S3: the signed API re-read (whole object up to 16 MiB re-hashed; larger: size plus + * byte-exact head and tail windows), then an anonymous ranged read of the head and tail + * through the public URL — the one readers use; + * - IPFS: kubo's CID must equal the local derivation and be pinned, then a re-read through the + * node's gateway when one is configured, then an anonymous read through the public gateway + * when one is recorded. + * Every URL recorded must be a public https URL (`lib/net.ts`); a profile whose public address + * is not is refused before any upload. */ import type { EvoSDK } from '@dashevo/evo-sdk' import type { WriteAuth } from '../sdk' import { estimateChunkCredits } from '../sdk/cost' +import { isRecordableUri } from '../net' import type { V2RepoRef } from '../repo/contract' import { manifestUrisProblem, putPlatformChunks } from '../repo/push' import { addVerified, gatewayUrl, remotePin } from './ipfs' -import { artifactKey, type StoragePolicy, type StorageProfile } from './profiles' -import { getObject, headObject, publicObjectUrl, putObject, s3Uri, type S3Settings } from './s3' +import { artifactKey, profileProblem, publishProblem, type StoragePolicy, type StorageProfile } from './profiles' +import { getObject, getPublic, headObject, putObject, publicObjectUrl, s3Uri, type S3Settings } from './s3' import { sha256Hex } from './sigv4' -import { bytesEqual, errText } from './util' +import { bytesEqual, errText, timedFetch } from './util' /** Artifacts up to this size are verified by a full re-download and SHA-256. */ export const FULL_VERIFY_MAX = 16 * 1024 * 1024 @@ -50,7 +57,7 @@ export interface StoredArtifact { /** 0 when a Platform copy exists (chunks), else 1. */ readonly storage: 0 | 1 readonly chunkCount: number - /** `platform://` first, then public http(s) URLs, then the rest (`ipfs://`, `s3://`). */ + /** `platform://` first, then public https URLs, then the rest (`ipfs://`, `s3://`). */ readonly uris: readonly string[] /** Targets that confirmed, in policy order. */ readonly confirmed: readonly string[] @@ -61,7 +68,7 @@ export interface StoredArtifact { export interface PlatformQuestion { readonly bytes: number readonly estimateCredits: number - /** Why Platform is being offered: no storage configured, or the policy was not met. */ + /** Why Platform is being used: a target of the policy, the fallback, or nothing configured. */ readonly reason: string } @@ -71,11 +78,15 @@ export class ReplicationError extends Error { readonly required: number, readonly confirmed: readonly string[], readonly failures: readonly TargetFailure[], + /** Whether any Platform chunks were written before the policy failed (they stay: chunks are permanent). */ + readonly platformWritten: boolean, ) { super( `storage policy not met: ${confirmed.length} of ${required} required target(s) confirmed` + failures.map((f) => `; ${f.target}: ${f.reason}`).join('') + - '. Nothing was written to Platform.', + (platformWritten + ? '. Some pack chunks were written to Platform before the push stopped; they are kept, and a retry does not pay for them again. No manifest or ref was written.' + : '. Nothing was written to Platform.'), ) this.name = 'ReplicationError' } @@ -89,7 +100,16 @@ export class PlatformDeclinedError extends Error { } } -/** Re-read an S3 copy and check it is exactly `bytes`. */ +/** The byte windows compared at both ends of an artifact (the whole of a small one). */ +function edgeWindows(size: number): (readonly [number, number])[] { + if (size <= 2 * EDGE_WINDOW) return [[0, size]] + return [ + [0, EDGE_WINDOW], + [size - EDGE_WINDOW, size], + ] +} + +/** Re-read an S3 copy through the signed API and check it is exactly `bytes`. */ async function verifyS3(s: S3Settings, p: StorageProfile, key: string, bytes: Uint8Array, hashHex: string): Promise { if (bytes.length <= FULL_VERIFY_MAX) { const got = await getObject(s, p.secrets, key) @@ -98,17 +118,31 @@ async function verifyS3(s: S3Settings, p: StorageProfile, key: string, bytes: Ui } const size = await headObject(s, p.secrets, key) if (size !== bytes.length) throw new Error(`re-read of ${key}: object missing or wrong size (${size} ≠ ${bytes.length})`) - for (const [start, end] of [ - [0, Math.min(EDGE_WINDOW, bytes.length)], - [Math.max(0, bytes.length - EDGE_WINDOW), bytes.length], - ] as const) { + for (const [start, end] of edgeWindows(bytes.length)) { const got = await getObject(s, p.secrets, key, `bytes=${start}-${end - 1}`) if (!bytesEqual(got, bytes.subarray(start, end))) throw new Error(`re-read of ${key} bytes ${start}..${end} differ from the artifact`) } } +/** Read the head and tail of a copy anonymously, the way every reader will. */ +async function verifyPublic(label: string, read: (range: string) => Promise<{ status: number; bytes: Uint8Array }>, bytes: Uint8Array): Promise { + for (const [start, end] of edgeWindows(bytes.length)) { + let got: { status: number; bytes: Uint8Array } + try { + got = await read(`bytes=${start}-${end - 1}`) + } catch (e) { + throw new Error(`${label} is not readable by others from a browser (${errText(e)}): check public access and CORS`) + } + if (got.status !== 206 && got.status !== 200) throw new Error(`${label} answered HTTP ${got.status} to an anonymous read: the copy is not public`) + const slice = got.status === 200 ? got.bytes.subarray(start, end) : got.bytes + if (!bytesEqual(slice, bytes.subarray(start, end))) throw new Error(`${label} served bytes ${start}..${end} that differ from the artifact`) + } +} + /** Store and verify on one external profile; the URIs its manifest entry records. */ async function storeExternal(p: StorageProfile, bytes: Uint8Array, hashHex: string): Promise { + const problem = profileProblem(p) ?? publishProblem(p) + if (problem) throw new Error(`the profile is not usable: ${problem}`) const s = p.settings if (s.kind === 's3') { const key = artifactKey(s, hashHex) @@ -126,18 +160,26 @@ async function storeExternal(p: StorageProfile, bytes: Uint8Array, hashHex: stri throw new Error(`${errText(second)} (also after a re-upload; first attempt: ${errText(first)})`) } } + await verifyPublic(`the public URL (${new URL(s.publicUrl).host})`, (range) => getPublic(s, key, range), bytes) return [publicObjectUrl(s, key), s3Uri(s, key)] } if (s.kind === 'ipfs-kubo' || s.kind === 'ipfs-pinning-service') { const cid = await addVerified(s, p.secrets, bytes) if (s.kind === 'ipfs-pinning-service') await remotePin(s, p.secrets, cid, `dash-forge ${hashHex.slice(0, 16)}`) if (s.gateway !== '') { - const resp = await fetch(gatewayUrl(s.gateway, cid), { credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(60_000) }) - const got = new Uint8Array(await resp.arrayBuffer()) - if (!resp.ok || (await sha256Hex(got)) !== hashHex) throw new Error(`re-read of ipfs://${cid} through the gateway did not return the artifact`) + const r = await timedFetch(gatewayUrl(s.gateway, cid), { credentials: 'omit', cache: 'no-store' }) + const got = await r.bytes() + if (!r.resp.ok || (await sha256Hex(got)) !== hashHex) throw new Error(`re-read of ipfs://${cid} through the gateway did not return the artifact`) } const uris = [`ipfs://${cid}`] - if (s.publicGateway !== '') uris.unshift(gatewayUrl(s.publicGateway, cid)) + if (s.publicGateway !== '') { + const url = gatewayUrl(s.publicGateway, cid) + await verifyPublic(`the public gateway (${new URL(url).host})`, async (range) => { + const r = await timedFetch(url, { headers: { Range: range }, credentials: 'omit', cache: 'no-store' }) + return { status: r.resp.status, bytes: await r.bytes() } + }, bytes) + uris.unshift(url) + } return uris } throw new Error('not an external profile') @@ -145,7 +187,7 @@ async function storeExternal(p: StorageProfile, bytes: Uint8Array, hashHex: stri /** * Every recorded URI, de-duplicated: the `platform://` locator first (released helpers read - * `uris[0]` of a storage-0 manifest as the chunk locator), then public http(s) URLs, then the + * `uris[0]` of a storage-0 manifest as the chunk locator), then public https URLs, then the * rest — each group in policy order (parity with forge-core `Replication::uris`). */ export function orderUris(groups: readonly (readonly string[])[]): string[] { @@ -163,8 +205,12 @@ export function orderUris(groups: readonly (readonly string[])[]): string[] { /** * Trim to the manifest's `uris` budget (8 × 300 bytes): private `s3://` locators go first * (readers without that profile cannot use them); then an error, never a silent truncation. + * A URI no reader may fetch (plain http, this machine, a private network) is refused outright: + * it would be published on chain for every reader's browser to request. */ export function fitManifestUris(uris: readonly string[]): string[] { + const bad = uris.find((u) => !isRecordableUri(u)) + if (bad !== undefined) throw new Error(`refusing to record ${bad} on chain: only public https addresses can be read by others`) const fits = (list: readonly string[]): boolean => manifestUrisProblem(list) === null if (fits(uris)) return [...uris] const noS3 = uris.filter((u) => !u.startsWith('s3://')) @@ -176,15 +222,16 @@ export function fitManifestUris(uris: readonly string[]): string[] { export interface StoreOptions { readonly policy: StoragePolicy | null readonly profiles: readonly StorageProfile[] + /** Asked once, with the price, before anything is written to Platform. False stops it. */ readonly confirmPlatform: (q: PlatformQuestion) => Promise readonly onStep?: (e: UploadEvent) => void } /** * Store `bytes` under `policy`, verifying every copy, and return what the manifest records. - * With no policy, or when the policy is not met and it allows the Platform fallback, the user - * is asked (with the price) before any chunk is written; declining throws - * {@link PlatformDeclinedError}. Anything else short of `replicas` throws {@link ReplicationError}. + * Before the first Platform chunk the user is asked, with the price; declining throws + * {@link PlatformDeclinedError} (nothing configured) or {@link ReplicationError} (the policy + * needed the Platform copy). Fewer than `replicas` confirmations throws {@link ReplicationError}. */ export async function storeArtifact( sdk: EvoSDK, @@ -198,18 +245,28 @@ export async function storeArtifact( const confirmed: { target: string; uris: string[]; platform: boolean }[] = [] const failures: TargetFailure[] = [] let chunkCount = 0 + let platformWritten = false + let platformAgreed: boolean | null = null const fail = (target: string, e: unknown): void => { failures.push({ target, reason: errText(e) }) step({ target, phase: 'failed', reason: errText(e) }) } - const askPlatform = (reason: string): Promise => - opts.confirmPlatform({ bytes: bytes.length, estimateCredits: estimateChunkCredits(bytes.length), reason }) + /** Ask once per push; every Platform write goes through this first. */ + const agreePlatform = async (reason: string): Promise => { + if (platformAgreed === null) { + platformAgreed = await opts.confirmPlatform({ bytes: bytes.length, estimateCredits: estimateChunkCredits(bytes.length), reason }) + } + return platformAgreed + } const platformCopy = async (target: string): Promise => { step({ target, phase: 'start' }) try { - const r = await putPlatformChunks(sdk, auth, repo, bytes, hashHex, (done, total) => step({ target, phase: 'progress', done, total })) + const r = await putPlatformChunks(sdk, auth, repo, bytes, hashHex, (done, total) => { + platformWritten = true + step({ target, phase: 'progress', done, total }) + }) chunkCount = r.chunkCount confirmed.push({ target, uris: [r.locator], platform: true }) step({ target, phase: 'done', uris: [r.locator] }) @@ -221,7 +278,7 @@ export async function storeArtifact( const policy = opts.policy if (policy === null || policy.targets.length === 0) { - if (!(await askPlatform('No storage is configured for browser pushes to this repo.'))) throw new PlatformDeclinedError() + if (!(await agreePlatform('No storage is configured for browser pushes to this repo.'))) throw new PlatformDeclinedError() await platformCopy('platform') } else { const byName = new Map(opts.profiles.map((p) => [p.name, p])) @@ -241,17 +298,25 @@ export async function storeArtifact( } }), ) - // Platform targets only if they can still make up the policy: never pay for chunks - // whose push is about to fail anyway. + const notMet = (): ReplicationError => new ReplicationError(policy.replicas, confirmed.map((c) => c.target), failures, platformWritten) + // Platform targets only if they can still make up the policy: never pay for chunks whose + // push is about to fail anyway — and never without the user's go-ahead on the price. for (const [i, { name }] of onChain.entries()) { if (confirmed.length + (onChain.length - i) < policy.replicas) break + if (!(await agreePlatform(`Your storage policy for this repo includes Dash Platform (${name}).`))) { + fail(name, new Error('storing on Platform was declined')) + throw notMet() + } await platformCopy(name).catch(() => undefined) } if (confirmed.length < policy.replicas) { - const notMet = (): ReplicationError => new ReplicationError(policy.replicas, confirmed.map((c) => c.target), failures) if (!policy.platformFallback || confirmed.some((c) => c.platform)) throw notMet() - if (!(await askPlatform(`Your storage did not confirm (${failures.map((f) => f.target).join(', ')}).`))) throw notMet() - await platformCopy('platform (fallback)') + if (!(await agreePlatform(`Your storage did not confirm (${failures.map((f) => f.target).join(', ')}).`))) throw notMet() + try { + await platformCopy('platform (fallback)') + } catch { + throw notMet() + } } } diff --git a/forge-web/lib/storage/util.ts b/forge-web/lib/storage/util.ts index b257a2003..c4dfecee6 100644 --- a/forge-web/lib/storage/util.ts +++ b/forge-web/lib/storage/util.ts @@ -9,3 +9,103 @@ export function bytesEqual(a: Uint8Array, b: Uint8Array): boolean { export function errText(e: unknown): string { return e instanceof Error ? e.message : String(e) } + +/** A response is expected to begin within this long (a dead or silent host). */ +export const CONNECT_MS = 15_000 +/** A response body may pause this long between chunks before it is abandoned. */ +export const IDLE_MS = 120_000 +/** + * The slowest upload rate assumed when a request carries a body: `fetch` reports no upload + * progress, so the wait for the response grows with the body instead of being one fixed cap. + * 16 KiB/s is well below any usable uplink; a stall still ends within `CONNECT_MS + IDLE_MS` + * plus the body at that rate. + */ +const MIN_UPLOAD_BYTES_PER_S = 16 * 1024 + +/** The request timed out (no answer, or the body stalled). */ +export class TimeoutError extends Error { + constructor(message: string) { + super(message) + this.name = 'TimeoutError' + } +} + +/** A response whose body is read with an idle deadline re-armed on every chunk. */ +export interface TimedResponse { + readonly resp: Response + bytes(): Promise + text(): Promise +} + +/** + * `fetch` with progress deadlines rather than one total cap (forge-core's clients use a 15 s + * connect and a 120 s read timeout): the response must begin within {@link CONNECT_MS} (plus + * the upload time of `uploadBytes` at a conservative rate), then the body may stream for as + * long as it keeps moving, pausing at most {@link IDLE_MS}. + */ +export async function timedFetch( + url: string | URL, + init: RequestInit, + opts: { readonly uploadBytes?: number; readonly signal?: AbortSignal; readonly idleMs?: number } = {}, +): Promise { + const controller = new AbortController() + const idleMs = opts.idleMs ?? IDLE_MS + let timedOut: string | null = null + let timer: ReturnType | undefined + const arm = (ms: number, why: string): void => { + clearTimeout(timer) + timer = setTimeout(() => { + timedOut = why + controller.abort() + }, ms) + } + const onCancel = (): void => controller.abort() + opts.signal?.addEventListener('abort', onCancel) + const uploadMs = Math.ceil(((opts.uploadBytes ?? 0) / MIN_UPLOAD_BYTES_PER_S) * 1000) + arm(CONNECT_MS + uploadMs, `no answer within ${Math.round((CONNECT_MS + uploadMs) / 1000)}s`) + const done = (): void => { + clearTimeout(timer) + opts.signal?.removeEventListener('abort', onCancel) + } + let resp: Response + try { + resp = await fetch(url, { ...init, signal: controller.signal }) + } catch (e) { + done() + if (timedOut !== null) throw new TimeoutError(timedOut) + throw e + } + const bytes = async (): Promise => { + try { + if (resp.body === null) return new Uint8Array(await resp.arrayBuffer()) + const reader = resp.body.getReader() + const parts: Uint8Array[] = [] + let total = 0 + for (;;) { + arm(idleMs, `the answer stalled for ${idleMs / 1000}s`) + const { done: end, value } = await reader.read() + if (end) break + parts.push(value) + total += value.length + } + const out = new Uint8Array(total) + let at = 0 + for (const p of parts) { + out.set(p, at) + at += p.length + } + return out + } catch (e) { + if (timedOut !== null) throw new TimeoutError(timedOut) + throw e + } finally { + done() + } + } + return { resp, bytes, text: async () => new TextDecoder().decode(await bytes()) } +} + +/** Printable ASCII only: header values (`fetch` throws on anything else, quoting the value). */ +export function isHeaderSafe(value: string): boolean { + return /^[\x20-\x7e]*$/.test(value) +} diff --git a/forge-web/lib/view/browse-fallback.test.ts b/forge-web/lib/view/browse-fallback.test.ts index a59f36fd7..3ec6cee95 100644 --- a/forge-web/lib/view/browse-fallback.test.ts +++ b/forge-web/lib/view/browse-fallback.test.ts @@ -163,7 +163,7 @@ describe('startFallback with external-storage packs', () => { const external = manifestFor(ext.pack, 1, { storage: 1, chunkCount: 0, - uris: ['http://127.0.0.1:9000/forge-byo/pack', 's3://forge-byo/pack'], + uris: ['https://mirror.example/forge-byo/pack', 's3://forge-byo/pack'], createdAt: 2, documentId: 'b', uploader: 'owner', @@ -172,16 +172,16 @@ describe('startFallback with external-storage packs', () => { const calls = stubFetch({}) const ctx = await startFallback(mockSdk(new Map([[platform.packHash, plat.pack]])), repo, [platform, external]) - expect(calls).toEqual(['http://127.0.0.1:9000/forge-byo/pack']) // s3:// is not browser-fetchable + expect(calls).toEqual(['https://mirror.example/forge-byo/pack']) // s3:// is not browser-fetchable expect(ctx.unavailable).toHaveLength(1) expect(ctx.unavailable?.[0]?.packHash).toBe(external.packHash) - expect(ctx.unavailable?.[0]?.hosts).toEqual(['127.0.0.1:9000']) + expect(ctx.unavailable?.[0]?.hosts).toEqual(['mirror.example']) expect(Array.from((await ctx.reader.readObject(plat.oid)).bytes)).toEqual( Array.from(new TextEncoder().encode('on-chain content\n')), ) // A needed object from the skipped pack gets a per-view error naming pack and storage. await expect(ctx.reader.readObject(ext.oid)).rejects.toThrow( - new RegExp(`${external.packHash.slice(0, 12)}.*127\\.0\\.0\\.1:9000`), + new RegExp(`${external.packHash.slice(0, 12)}.*mirror\\.example`), ) // The trust ledger reports the gap: content is partial, never verified. const checks = contentChecks(repo.contractId) @@ -234,7 +234,7 @@ describe('startFallback with external-storage packs', () => { const ext = blobPack('gone\n') const platform = manifestFor(plat.pack, 1, { createdAt: 1, documentId: 'a' }) const external = (documentId: string): PackManifest => - manifestFor(ext.pack, 1, { storage: 1, uris: ['http://127.0.0.1:9000/p'], createdAt: 2, documentId }) + manifestFor(ext.pack, 1, { storage: 1, uris: ['https://mirror.example/p'], createdAt: 2, documentId }) stubFetch({}) const ctx = await startFallback( mockSdk(new Map([[platform.packHash, plat.pack]])), @@ -283,7 +283,7 @@ describe('startFallback with external-storage packs', () => { it('fails with the reasons when no live pack at all could be fetched', async () => { const ext = blobPack('x\n') - const external = manifestFor(ext.pack, 1, { storage: 1, uris: ['http://127.0.0.1:9000/p'] }) + const external = manifestFor(ext.pack, 1, { storage: 1, uris: ['https://mirror.example/p'] }) stubFetch({}) await expect( startFallback(mockSdk(new Map()), { kind: 'v1', contractId: 'fallback-none', ownerId: 'owner', name: '' }, [external]), @@ -338,13 +338,13 @@ describe('startFallback with external-storage packs', () => { const platform = manifestFor(plat.pack, 1, { createdAt: 0, documentId: 'a' }) const dead = (i: number): PackManifest => { const p = blobPack(`gone ${i}\n`) - return manifestFor(p.pack, 1, { storage: 1, uris: ['http://127.0.0.1:9000/p'], createdAt: i + 1, documentId: `x${i}` }) + return manifestFor(p.pack, 1, { storage: 1, uris: ['https://mirror.example/p'], createdAt: i + 1, documentId: `x${i}` }) } const calls = stubFetch({}) const sdk = mockSdk(new Map([[platform.packHash, plat.pack]])) await startFallback(sdk, { kind: 'v1', contractId: 'fallback-dead', ownerId: 'o', name: '' }, [platform, dead(1)]) await startFallback(sdk, { kind: 'v1', contractId: 'fallback-dead-2', ownerId: 'o', name: '' }, [platform, dead(2)]) - expect(calls).toEqual(['http://127.0.0.1:9000/p']) + expect(calls).toEqual(['https://mirror.example/p']) }) it('cancels external downloads when an on-chain pack fails', async () => { @@ -378,12 +378,12 @@ describe('startFallback with external-storage packs', () => { const packA = packFrame(concat(objHeader(T_BLOB, base.length), zlibSync(base))) const delta = copyInsertDelta(base.length, target.length, 40, new TextEncoder().encode('cat\n')) const packB = packFrame(concat(objHeader(T_REF_DELTA, delta.length), hexToBytes(baseOid), zlibSync(delta))) - const a = manifestFor(packA, 1, { storage: 1, uris: ['http://127.0.0.1:9000/a'], createdAt: 0, documentId: 'a' }) + const a = manifestFor(packA, 1, { storage: 1, uris: ['https://mirror.example/a'], createdAt: 0, documentId: 'a' }) const b = manifestFor(packB, 1, { createdAt: 1, documentId: 'b' }) stubFetch({}) await expect( startFallback(mockSdk(new Map([[b.packHash, packB]])), { kind: 'v1', contractId: 'fallback-thin', ownerId: 'o', name: '' }, [a, b]), - ).rejects.toThrow(new RegExp(`${a.packHash.slice(0, 12)}.*127\\.0\\.0\\.1:9000`)) + ).rejects.toThrow(new RegExp(`${a.packHash.slice(0, 12)}.*mirror\\.example`)) }) it('expires a partial clone so a later view retries the mirrors', async () => { diff --git a/forge-web/lib/view/browse-source.ts b/forge-web/lib/view/browse-source.ts index 4aee1e3a8..0c877f961 100644 --- a/forge-web/lib/view/browse-source.ts +++ b/forge-web/lib/view/browse-source.ts @@ -42,6 +42,7 @@ import { type V2RepoRef, } from '../repo' import { base64ToBytes, queryDocumentsWithProof } from '../sdk' +import { isPublicHttpsUrl } from '../net' import { externalSourceName, noteContentCheck, objectObserver } from './content-checks' import { describePack, readGateways } from './storage-status' @@ -286,8 +287,10 @@ export function externalFetchUrls( ): string[] { const out: string[] = [] for (const uri of uris) { - if (/^https?:\/\//i.test(uri)) out.push(uri) - const ipfs = /^ipfs:\/\/(.+)$/i.exec(uri) + // A manifest is written by whoever pushed: a URL at this machine, a private network or + // over plain http would have every reader's browser request its own local services. + if (isPublicHttpsUrl(uri)) out.push(uri) + const ipfs = /^ipfs:\/\/([A-Za-z0-9]+)$/.exec(uri) if (ipfs !== null) { for (const gw of gateways) out.push(`${gw.replace(/\/+$/, '')}/ipfs/${ipfs[1]}`) } From e9be26faea909fe3aaad60d454e6a714812b8072 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 26 Sep 2026 17:28:52 -0500 Subject: [PATCH 4/5] fix(web): use the AA text shades for storage status words Amber and green status text in the storage wizard now pairs the -700 shade on light surfaces with the base on dark, matching the Verification card. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/components/storage/copy-block.tsx | 2 +- forge-web/components/storage/profile-form.tsx | 2 +- forge-web/components/storage/storage-test.tsx | 4 ++-- forge-web/components/storage/storage-wizard.tsx | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/forge-web/components/storage/copy-block.tsx b/forge-web/components/storage/copy-block.tsx index 158dc2f76..c635faf18 100644 --- a/forge-web/components/storage/copy-block.tsx +++ b/forge-web/components/storage/copy-block.tsx @@ -26,7 +26,7 @@ export function CopyBlock({ text, label }: { text: string; label: string }): JSX aria-label={label} className="absolute right-2 top-2 rounded p-1 text-anvil-500 hover:bg-anvil-100 hover:text-anvil-800 dark:text-anvil-400 dark:hover:bg-anvil-800 dark:hover:text-anvil-100" > - {copied ? : } + {copied ? : } ) diff --git a/forge-web/components/storage/profile-form.tsx b/forge-web/components/storage/profile-form.tsx index 57aed7f78..8962bfb2d 100644 --- a/forge-web/components/storage/profile-form.tsx +++ b/forge-web/components/storage/profile-form.tsx @@ -161,7 +161,7 @@ export function ProfileForm({
{preset.kind === 'ipfs-kubo' || preset.kind === 'ipfs-pinning-service' ? (
- + kubo’s RPC API is the node’s admin interface. Letting this web app call it means trusting this site with your node, unless kubo limits the token it uses. Create a token restricted to add, pin-check, unpin and id (kubo API.Authorizations with AllowedPaths; the test prints the commands if the node refuses this origin) and paste it as the API Authorization header. diff --git a/forge-web/components/storage/storage-test.tsx b/forge-web/components/storage/storage-test.tsx index 1ec03f9ef..93ad00934 100644 --- a/forge-web/components/storage/storage-test.tsx +++ b/forge-web/components/storage/storage-test.tsx @@ -26,7 +26,7 @@ import { cn } from '@/lib/utils' const META: Readonly> = { pending: { word: 'waiting', klass: 'text-anvil-500 dark:text-anvil-400', Icon: CircleDashed }, running: { word: 'testing', klass: 'text-anvil-500 dark:text-anvil-400', Icon: Loader2 }, - ok: { word: 'ok', klass: 'text-verify', Icon: CheckCircle2 }, + ok: { word: 'ok', klass: 'text-verify-700 dark:text-verify', Icon: CheckCircle2 }, fail: { word: 'FAIL', klass: 'text-danger-700 dark:text-danger-400', Icon: XCircle }, skipped: { word: 'skipped', klass: 'text-anvil-500 dark:text-anvil-400', Icon: MinusCircle }, } @@ -117,7 +117,7 @@ export function StorageTest({ profile, onDone }: { profile: StorageProfile; onDo {corsFailed && fix.text ? (

- + Fix CORS. {fix.where} diff --git a/forge-web/components/storage/storage-wizard.tsx b/forge-web/components/storage/storage-wizard.tsx index 1cea62938..e1dfbd03c 100644 --- a/forge-web/components/storage/storage-wizard.tsx +++ b/forge-web/components/storage/storage-wizard.tsx @@ -63,7 +63,7 @@ export function StorageWizard(): JSX.Element { return (

{!storable ? ( -

+

This tab signs with a pasted key, which has no vault. Storage settings are kept encrypted with a stored key: sign in by importing your identity to save them.

) : null} @@ -268,7 +268,7 @@ function AddProfile({ Save profile - + {status}
@@ -311,7 +311,7 @@ function Profiles({ config, storable, save, onEdit }: { config: StorageConfig; s {p.name} {where} {test ? ( - + {test.ok ? 'passed' : 'failed some checks'} · {formatDate(test.at)} ) : p.settings.kind === 'platform' ? null : ( @@ -420,7 +420,7 @@ function DefaultPolicy({ config, storable, save }: { config: StorageConfig; stor > Save default - + {saved ? 'Saved.' : problem && targets.length > 0 ? problem : ''} {err ? {err} : null} From 3b8d715363f86e39c6e2c7f7bf840a6f26fb0dd2 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 26 Sep 2026 17:37:07 -0500 Subject: [PATCH 5/5] fix(web): address the storage review follow-ups A local kubo or MinIO works in every build: the CSP allows loopback http again (readers fetch only public https URLs themselves), and only the local API and endpoint may be loopback, never a published address. The kubo fix script works on a fresh node (the origin merge no longer relies on a failing config read) and creates a full-access owner token first, since any API.Authorizations entry locks the RPC API for the CLI and WebUI. Also: the chunks-written flag only after a chunk is written, request deadlines cleared when the body is not read, the test fingerprint hashed, AWS's bare wrong-region 301 told apart from CORS, and the renewal notice cleared once shown. Co-Authored-By: Claude Opus 5.5 (1M context) --- forge-web/app/layout.tsx | 12 ++++++------ forge-web/components/storage/storage-test.tsx | 5 ++++- forge-web/contexts/auth-context.tsx | 6 ++++-- forge-web/lib/auth/controller.ts | 5 +++++ forge-web/lib/net.ts | 12 ++++++------ forge-web/lib/repo/push.ts | 9 ++++++--- forge-web/lib/storage/cors.ts | 18 ++++++++++++------ forge-web/lib/storage/probe.ts | 18 +++++++++++++++--- forge-web/lib/storage/profiles.ts | 8 +++----- forge-web/lib/storage/s3.ts | 4 +++- forge-web/lib/storage/storage.test.ts | 17 +++++++++++------ forge-web/lib/storage/upload.ts | 4 ++-- forge-web/lib/storage/util.ts | 8 +++++++- 13 files changed, 84 insertions(+), 42 deletions(-) diff --git a/forge-web/app/layout.tsx b/forge-web/app/layout.tsx index ed9a4fc9c..b63743390 100644 --- a/forge-web/app/layout.tsx +++ b/forge-web/app/layout.tsx @@ -1,7 +1,6 @@ import type { Metadata } from 'next' import type { ReactNode } from 'react' import { Providers } from '@/components/providers' -import { LOCAL_HTTP_ALLOWED } from '@/lib/net' import './globals.css' export const metadata: Metadata = { @@ -17,10 +16,11 @@ export const metadata: Metadata = { // - frame-ancestors is ignored in a CSP; the host must send it as a header (GitHub // Pages cannot; see docs/guides/identity-and-keys.md). // - connect-src https:/wss:: DAPI endpoints + IPFS/S3/HTTPS pack backends. Plain http to this -// machine is allowed ONLY in devnet and dev builds (`LOCAL_HTTP_ALLOWED`: a local MinIO or -// kubo while developing); production builds never open it, for any page. Readers never -// fetch loopback or private hosts anyway (`lib/net.ts`), and nothing loopback is ever -// recorded on chain. +// machine is allowed so the storage settings can reach the user's OWN local node (a kubo +// RPC API, a MinIO endpoint), which is how kubo ships. It cannot be narrowed to one page: a +// static export has one CSP, and several combine as an intersection. It is not a read +// path: readers fetch only public https URLs (`lib/net.ts` `externalFetchUrls`), and nothing +// loopback or private is ever recorded on chain (`fitManifestUris`, `publishProblem`). // - worker-src blob:: materialization / search / pack workers run off-main-thread. const CSP = [ "default-src 'self'", @@ -28,7 +28,7 @@ const CSP = [ "style-src 'self' 'unsafe-inline'", "img-src 'self' data: https: blob:", "font-src 'self'", - `connect-src 'self' https: wss:${LOCAL_HTTP_ALLOWED ? ' http://127.0.0.1:* http://localhost:*' : ''}`, + "connect-src 'self' https: wss: http://127.0.0.1:* http://localhost:*", "worker-src 'self' blob:", "child-src 'self' blob:", "frame-ancestors 'none'", diff --git a/forge-web/components/storage/storage-test.tsx b/forge-web/components/storage/storage-test.tsx index 93ad00934..ff09171d8 100644 --- a/forge-web/components/storage/storage-test.tsx +++ b/forge-web/components/storage/storage-test.tsx @@ -7,6 +7,8 @@ */ import { useEffect, useRef, useState } from 'react' +import { sha256 } from '@noble/hashes/sha2.js' +import { bytesToHex } from '@noble/hashes/utils.js' import { AlertTriangle, CheckCircle2, CircleDashed, Loader2, MinusCircle, RotateCw, XCircle } from 'lucide-react' import { IPFS_ROWS, @@ -38,7 +40,8 @@ function rowsFor(p: StorageProfile): ProbeRow[] { /** A stable fingerprint of what a test ran against (settings and secrets). */ export function profileSnapshot(p: StorageProfile): string { - return JSON.stringify([p.name, p.settings, p.secrets]) + // A digest, not the values: it becomes a React key and sits in state, where dev tools show it. + return bytesToHex(sha256(new TextEncoder().encode(JSON.stringify([p.name, p.settings, p.secrets])))) } /** diff --git a/forge-web/contexts/auth-context.tsx b/forge-web/contexts/auth-context.tsx index a6863b7e0..23dd1c685 100644 --- a/forge-web/contexts/auth-context.tsx +++ b/forge-web/contexts/auth-context.tsx @@ -108,8 +108,10 @@ export function AuthProvider({ // A notice from a sign-in step (e.g. storage settings that could not survive a key renewal). const notice = state.notice ?? null useEffect(() => { - if (notice) toast({ title: 'Storage settings were not carried over', tone: 'warn', detail: notice }) - }, [notice]) + if (!notice) return + toast({ title: 'Storage settings were not carried over', tone: 'warn', detail: notice }) + controller.clearNotice() + }, [notice, controller]) const [vaults, setVaults] = useState([]) const reloadVaults = useCallback(() => { diff --git a/forge-web/lib/auth/controller.ts b/forge-web/lib/auth/controller.ts index bea56f125..58773a529 100644 --- a/forge-web/lib/auth/controller.ts +++ b/forge-web/lib/auth/controller.ts @@ -122,6 +122,11 @@ export class AuthController { return this.state } + /** The UI showed the notice: clear it, so the next one (even the same text) shows too. */ + clearNotice(): void { + if (this.state.notice) this.setState({ notice: null }) + } + subscribe(listener: Listener): () => void { this.listeners.add(listener) return () => { diff --git a/forge-web/lib/net.ts b/forge-web/lib/net.ts index 1d6899351..a301fc691 100644 --- a/forge-web/lib/net.ts +++ b/forge-web/lib/net.ts @@ -10,13 +10,13 @@ */ /** - * Whether this build may talk plain http to this machine (a local MinIO or kubo): devnet and - * dev builds only. The CSP (`app/layout.tsx`) and the storage profile checks both follow it, - * so a production build never opens loopback http for any page. + * Whether `hostname` names this machine or a private / link-local network. + * + * Literal addresses and reserved names only: a page cannot resolve DNS, so a public name that + * resolves to a private address (`127.0.0.1.nip.io`, DNS rebinding) passes. That residual is + * bounded by the rest of the design: every byte read is hash-checked, requests carry no + * credentials, and responses are opaque to other origins. */ -export const LOCAL_HTTP_ALLOWED: boolean = process.env.NEXT_PUBLIC_NETWORK === 'devnet' || process.env.NEXT_PUBLIC_FORGE_DEV === '1' - -/** Whether `hostname` names this machine or a private / link-local network. */ export function isPrivateHost(hostname: string): boolean { const h = hostname.toLowerCase().replace(/^\[|\]$/g, '') if (h === 'localhost' || h.endsWith('.localhost') || h.endsWith('.local') || h.endsWith('.internal')) return true diff --git a/forge-web/lib/repo/push.ts b/forge-web/lib/repo/push.ts index bd9e1a1eb..c408d963f 100644 --- a/forge-web/lib/repo/push.ts +++ b/forge-web/lib/repo/push.ts @@ -177,14 +177,16 @@ export async function putPlatformChunks( repo: V2RepoRef, bytes: Uint8Array, packHashHex: string, - onProgress?: (done: number, total: number) => void, + /** `written`: chunks this call itself created (0 on the first report, before any write). */ + onProgress?: (done: number, total: number, written: number) => void, ): Promise<{ locator: string; chunkCount: number }> { const chunks = splitChunks(bytes) const have = await storedSeqs(sdk, repo, auth.identityId, packHashHex, chunks.length) const R = decodeIdentifier(repo.repoId) const hash = hexToBytes(packHashHex) let done = have.size - onProgress?.(done, chunks.length) + let written = 0 + onProgress?.(done, chunks.length, written) for (const chunk of chunks) { if (have.has(chunk.seq)) continue const data: Record = { repoId: R, packHash: hash, seq: chunk.seq } @@ -198,11 +200,12 @@ export async function putPlatformChunks( data, intent: `chunk:${repo.repoId}:${packHashHex}:${chunk.seq}`, }) + written += 1 } catch (e) { if (!isDuplicate(e)) throw e } done += 1 - onProgress?.(done, chunks.length) + onProgress?.(done, chunks.length, written) } return { locator: `platform://${repo.forge.core}/${repo.repoId}/${auth.identityId}/${packHashHex}`, chunkCount: chunks.length } } diff --git a/forge-web/lib/storage/cors.ts b/forge-web/lib/storage/cors.ts index 48da82376..62295b85a 100644 --- a/forge-web/lib/storage/cors.ts +++ b/forge-web/lib/storage/cors.ts @@ -42,7 +42,7 @@ export function corsFix(provider: ProviderId, bucket: string, origin: string): C } case 'aws': return { - where: `Save as cors.json, then run: aws s3api put-bucket-cors --bucket ${b} --cors-configuration file://cors.json`, + where: `If the endpoint is not the bucket's own region, AWS answers with a redirect the browser cannot follow, which looks like a CORS failure: check the endpoint first (https://s3..amazonaws.com). Then save as cors.json and run: aws s3api put-bucket-cors --bucket ${b} --cors-configuration file://cors.json`, text: JSON.stringify({ CORSRules: s3Rules(origin) }, null, 2), } case 'b2': @@ -83,12 +83,18 @@ export function corsFix(provider: ProviderId, bucket: string, origin: string): C export function kuboCorsLines(origin: string): string { const paths = JSON.stringify(FORGE_RPC_PATHS) return [ - '# 1. A token that can only add, pin-check, unpin and identify (not the whole admin API):', - `TOKEN=$(openssl rand -hex 24)`, + '# 1. Once any API.Authorizations entry exists, kubo refuses EVERY RPC call without a token,', + '# your own `ipfs` commands and IPFS WebUI included. So first an owner token with full', + '# access (keep it; use it as `ipfs --api-auth bearer:$OWNER `), then one for this app', + '# that can only add, pin-check, unpin and identify:', + `OWNER=$(openssl rand -hex 24); TOKEN=$(openssl rand -hex 24)`, + `ipfs config --json API.Authorizations.owner "{\\"AuthSecret\\": \\"bearer:$OWNER\\", \\"AllowedPaths\\": [\\"/api/v0\\"]}"`, `ipfs config --json API.Authorizations.dash-forge "{\\"AuthSecret\\": \\"bearer:$TOKEN\\", \\"AllowedPaths\\": ${paths.replace(/"/g, '\\"')}}"`, - 'echo "API Authorization for the profile: Bearer $TOKEN"', - '# 2. Allow this app (keeping the origins already allowed):', - `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin "$(ipfs config API.HTTPHeaders.Access-Control-Allow-Origin 2>/dev/null | jq -c '(. // []) + ["${origin}"] | unique' || echo '["${origin}"]')"`, + 'echo "Owner token (for your own ipfs CLI): bearer:$OWNER"', + 'echo "API Authorization for the Forge profile: Bearer $TOKEN"', + '# 2. Allow this app, keeping the origins already allowed (none on a fresh node):', + `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin "$( (ipfs config API.HTTPHeaders.Access-Control-Allow-Origin 2>/dev/null || echo null) | jq -c '(. // []) + ["${origin}"] | unique')"`, + `# (no jq? ipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin '["${origin}"]' replaces the list instead)`, `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Methods '["POST"]'`, `ipfs config --json API.HTTPHeaders.Access-Control-Allow-Headers '["Authorization"]'`, '# 3. Let any origin read the gateway (the content is public):', diff --git a/forge-web/lib/storage/probe.ts b/forge-web/lib/storage/probe.ts index 97278daa8..de3347bf6 100644 --- a/forge-web/lib/storage/probe.ts +++ b/forge-web/lib/storage/probe.ts @@ -62,11 +62,23 @@ async function reachable(url: string | URL): Promise { } } -/** Whether `url` answers with a redirect (opaque, so only its type is visible). */ +/** + * Whether `url` answers with a redirect. A redirect with a `Location` shows as an opaque + * redirect; AWS's wrong-region answer is a bare `301 PermanentRedirect` with no `Location`, + * which fetch hands back as an ordinary response, visible only through a CORS read of its + * status (a `no-cors` one is opaque) — so both are tried. + */ async function redirects(url: string | URL): Promise { + const opts = { redirect: 'manual' as const, credentials: 'omit' as const, cache: 'no-store' as const } + try { + const r = await fetch(url, { ...opts, mode: 'no-cors', signal: AbortSignal.timeout(10_000) }) + if (r.type === 'opaqueredirect') return true + } catch { + return false + } try { - const r = await fetch(url, { mode: 'no-cors', redirect: 'manual', credentials: 'omit', cache: 'no-store', signal: AbortSignal.timeout(10_000) }) - return r.type === 'opaqueredirect' + const r = await fetch(url, { ...opts, signal: AbortSignal.timeout(10_000) }) + return r.type === 'opaqueredirect' || (r.status >= 300 && r.status < 400) } catch { return false } diff --git a/forge-web/lib/storage/profiles.ts b/forge-web/lib/storage/profiles.ts index 6a40bd221..6b695b59b 100644 --- a/forge-web/lib/storage/profiles.ts +++ b/forge-web/lib/storage/profiles.ts @@ -12,7 +12,7 @@ import { z } from 'zod' -import { LOCAL_HTTP_ALLOWED, isPublicHttpsUrl } from '../net' +import { isPublicHttpsUrl } from '../net' import { isHeaderSafe } from './util' /** The kinds a profile can be (parity with forge-core `Profile`). */ @@ -208,12 +208,10 @@ function checkUrl(field: string, value: string, use: UrlUse, opts: { originOnly? if (opts.originOnly && url.pathname !== '/' && url.pathname !== '') return `${field} must be an origin (scheme://host[:port]) with no path` // A published URL may point at this machine while testing (a local MinIO): the test runs and // its public row fails with the reason ({@link publishProblem}); uploads refuse it. - if (url.protocol === 'http:' && !(LOCAL_HTTP_ALLOWED && isLoopback(url.hostname))) { + if (url.protocol === 'http:' && !isLoopback(url.hostname)) { return use === 'published' ? `${field} must be https: it is recorded on chain for everyone to read` - : LOCAL_HTTP_ALLOWED - ? `${field} must be https (plain http is allowed only for a node on this machine, at 127.0.0.1 or localhost)` - : `${field} must be https` + : `${field} must be https (plain http is allowed only for a node on this machine, at 127.0.0.1 or localhost)` } return null } diff --git a/forge-web/lib/storage/s3.ts b/forge-web/lib/storage/s3.ts index 3f8a8eea8..6e3370380 100644 --- a/forge-web/lib/storage/s3.ts +++ b/forge-web/lib/storage/s3.ts @@ -163,8 +163,9 @@ export async function getObject(s: S3Settings, secrets: ProfileSecrets, key: str /** Signed HEAD: the object's size, or null when it does not exist. */ export async function headObject(s: S3Settings, secrets: ProfileSecrets, key: string): Promise { const r = await send(s, secrets, 'signed HEAD', 'HEAD', key) + if (!r.resp.ok && r.resp.status !== 404) throw await failed('signed HEAD', r) + r.discard() if (r.resp.status === 404) return null - if (!r.resp.ok) throw await failed('signed HEAD', r) const len = Number(r.resp.headers.get('content-length')) return Number.isFinite(len) ? len : 0 } @@ -173,4 +174,5 @@ export async function headObject(s: S3Settings, secrets: ProfileSecrets, key: st export async function deleteObject(s: S3Settings, secrets: ProfileSecrets, key: string): Promise { const r = await send(s, secrets, 'delete', 'DELETE', key) if (!r.resp.ok && r.resp.status !== 404) throw await failed('delete', r) + r.discard() } diff --git a/forge-web/lib/storage/storage.test.ts b/forge-web/lib/storage/storage.test.ts index 9ff03d165..d0ea4d71d 100644 --- a/forge-web/lib/storage/storage.test.ts +++ b/forge-web/lib/storage/storage.test.ts @@ -10,7 +10,7 @@ import { resetMemoryStores, idbEntries, idbGet, idbPut } from '../idb' import { lockVault, storeInVault, unlockWithPassphrase } from '../auth/vault' import { splitChunks, manifestUrisProblem } from '../repo/push' import { CHUNK_PAYLOAD_MAX, FIELD_MAX } from '../constants' -import { LOCAL_HTTP_ALLOWED, isPrivateHost, isPublicHttpsUrl } from '../net' +import { isPrivateHost, isPublicHttpsUrl } from '../net' import { externalFetchUrls } from '../view/browse-source' import { corsFix } from './cors' import { cidV1RawLeaves } from './cid' @@ -77,9 +77,10 @@ describe('profiles', () => { expect(bad({ endpoint: 'https://x.example/bucket' })).toMatch(/origin/) expect(bad({ endpoint: 'https://user:pw@x.example' })).toMatch(/user name/) expect(bad({ endpoint: 'http://minio.example.org' })).toMatch(/https/) - // Plain http to this machine is a dev/devnet-build allowance only (as is the CSP's). - if (LOCAL_HTTP_ALLOWED) expect(bad({ endpoint: 'http://127.0.0.1:9000' })).toBeNull() - else expect(bad({ endpoint: 'http://127.0.0.1:9000' })).toMatch(/https/) + // The API endpoint may be plain http on this machine (the user's own MinIO); what is + // published may not (the public URL, checked by publishProblem). + expect(bad({ endpoint: 'http://127.0.0.1:9000' })).toBeNull() + expect(bad({ endpoint: 'http://192.168.1.4:9000' })).toMatch(/https/) expect(bad({ bucket: 'Forge' })).toMatch(/lowercase/) expect(bad({ prefix: 'a/../b' })).toMatch(/segment/) expect(bad({ publicUrl: '' })).toMatch(/public URL/) @@ -279,8 +280,12 @@ describe('CORS fix blocks', () => { expect(where).toMatch(/admin interface/) expect(text).toContain('API.Authorizations.dash-forge') for (const path of ['/api/v0/add', '/api/v0/pin/ls', '/api/v0/pin/rm', '/api/v0/id', '/api/v0/version']) expect(text).toContain(path) - expect(text).toMatch(/\(\. \/\/ \[\]\) \+ \["https:\/\/forge\.dashhq\.org"\] \| unique/) - expect(text).not.toMatch(/Access-Control-Allow-Origin '\["https:\/\/forge\.dashhq\.org"\]'/) + expect(text).toMatch(/\(ipfs config API\.HTTPHeaders\.Access-Control-Allow-Origin 2>\/dev\/null \|\| echo null\) \| jq -c '\(\. \/\/ \[\]\) \+ \["https:\/\/forge\.dashhq\.org"\] \| unique'/) + const commands = text.split('\n').filter((l) => !l.startsWith('#')) + expect(commands.join('\n')).not.toMatch(/Access-Control-Allow-Origin '\["https:\/\/forge\.dashhq\.org"\]'/) + // Any Authorizations entry locks the RPC API for everyone else: an owner token comes first. + expect(text).toMatch(/API\.Authorizations\.owner .*\\"AllowedPaths\\": \[\\"\/api\/v0\\"\]/) + expect(text.indexOf('API.Authorizations.owner')).toBeLessThan(text.indexOf('API.Authorizations.dash-forge')) }) }) diff --git a/forge-web/lib/storage/upload.ts b/forge-web/lib/storage/upload.ts index 77e4f8a33..e35b13591 100644 --- a/forge-web/lib/storage/upload.ts +++ b/forge-web/lib/storage/upload.ts @@ -263,8 +263,8 @@ export async function storeArtifact( const platformCopy = async (target: string): Promise => { step({ target, phase: 'start' }) try { - const r = await putPlatformChunks(sdk, auth, repo, bytes, hashHex, (done, total) => { - platformWritten = true + const r = await putPlatformChunks(sdk, auth, repo, bytes, hashHex, (done, total, written) => { + if (written > 0) platformWritten = true step({ target, phase: 'progress', done, total }) }) chunkCount = r.chunkCount diff --git a/forge-web/lib/storage/util.ts b/forge-web/lib/storage/util.ts index c4dfecee6..b7fe4c01c 100644 --- a/forge-web/lib/storage/util.ts +++ b/forge-web/lib/storage/util.ts @@ -35,6 +35,8 @@ export interface TimedResponse { readonly resp: Response bytes(): Promise text(): Promise + /** Done without the body (a HEAD, a status-only answer): clear the deadline, drop the body. */ + discard(): void } /** @@ -102,7 +104,11 @@ export async function timedFetch( done() } } - return { resp, bytes, text: async () => new TextDecoder().decode(await bytes()) } + const discard = (): void => { + done() + resp.body?.cancel().catch(() => undefined) + } + return { resp, bytes, text: async () => new TextDecoder().decode(await bytes()), discard } } /** Printable ASCII only: header values (`fetch` throws on anything else, quoting the value). */