+ Your buckets and IPFS nodes for browser pushes, tested from this page, with their keys encrypted in this browser. +
+ + Storage settings → + ++ Settings / Storage +
++ We host nothing. Packs go to a bucket or IPFS node you own; Platform keeps only the signed manifest and refs (about 0.0003 DASH a push). Readers check every byte against its on-chain hash, so your storage provides availability, never trust. +
+
+
+ {text}
+
+
+ | Where | +One-time (Platform fees) | +Monthly | +50 MiB repo, 10 pushes/month | +
|---|---|---|---|
| {where} | +{once} | +{monthly} | +{example} | +
+ Platform figures are estimates from measured fees; bucket prices are the providers’ list prices, rounded. USD is indicative. +
+
+ {f.secret ? SECRET_NOTE : null}
+ {keepable ? ` ${KEEP_NOTE}` : null}
+ {f.secret && hint ? ' ' : null}
+ {hint ? (
+ <>
+ {hint.text}
+ {hint.href ? (
+ <>
+ {' '}
+
+ where to find this
{error}
+ if (!config) { + return ( ++ Sign in to choose where your browser pushes to this repo go. Settings stay in this browser. +
+ ) + } + if (config.profiles.length === 0) { + return ( ++ No storage set up in this browser: browser pushes here would store packs on Platform at ~0.28 DASH/MiB, asking first.{' '} + Set up storage → +
+ ) + } + const shown = targets ?? effective?.targets.slice() ?? [] + const shownChoice = choice ?? (effective ? choiceOf(effective) : 'one') + const policy = shown.length > 0 ? policyFor(shown, shownChoice) : null + const problem = policy ? policyProblem(config, policy) : null + + return ( +
+
+ {summary} +
+ {corsFailed && fix.text ? ( +
+
+ Reads are allowed from any origin (the objects are public); writes only from {origin}. Then press Re-test. +
++ This tab signs with a pasted key, which has no vault. Storage settings are kept encrypted with a stored key: sign in by importing your identity to save them. +
+ ) : null} + +{err}
: null} ++ Packs stored as Platform documents: permanent, readable by anyone, nothing to run. Priced at about 0.28 DASH per MiB, charged when you push, and never refunded (Platform storage cannot be deleted, so nobody can break a repo others depend on). Every Platform write asks first, with its price. +
+{saveError}
: null} +{saveError}
: null} ++ {error} +
+ ) : null} ++ No storage yet. Browser pushes (merges, forks) would store packs on Platform at ~0.28 DASH/MiB, asking first. +
+ ) : ( ++ The default for every repo you push to from this browser (merges, fork updates, release assets). A repo can override it under its Settings → Storage. The CLI keeps its own policy in git config. +
+ + +([^<]{1,64})<\/Code>/.exec(text)?.[1] ?? ''
+ } catch {
+ /* body unreadable */
+ }
+ return new S3Error(op, resp.status, `${op} failed: HTTP ${resp.status}${code ? ` ${code}` : ''}${statusHint(resp.status)}`)
+}
+
+/** What an S3 error status usually means, as a suffix for the message ('' when nothing useful). */
+function statusHint(status: number): string {
+ switch (status) {
+ case 403:
+ return ' — check the key, its bucket permissions, the region (R2 wants auto) and the clock'
+ case 404:
+ return ' — check the bucket name and endpoint'
+ case 301:
+ case 307:
+ return ' — the bucket lives in another region; set that endpoint'
+ default:
+ return ''
+ }
+}
+
+/**
+ * An anonymous GET of `key` through the bucket's public URL (what every reader uses), whole or
+ * a `Range`: the check that a copy is readable by anyone, not just by the credential holder.
+ */
+export async function getPublic(s: S3Settings, key: string, range?: string): Promise<{ status: number; bytes: Uint8Array; contentRange: string | null }> {
+ const r = await timedFetch(publicObjectUrl(s, key), { credentials: 'omit', cache: 'no-store', ...(range ? { headers: { Range: range } } : {}) })
+ return { status: r.resp.status, contentRange: r.resp.headers.get('content-range'), bytes: await r.bytes() }
+}
+
+/** Signed PUT of `bytes` at `key`. */
+export async function putObject(s: S3Settings, secrets: ProfileSecrets, key: string, bytes: Uint8Array, contentType = 'application/octet-stream', signal?: AbortSignal): Promise {
+ const r = await send(s, secrets, 'signed PUT', 'PUT', key, { body: bytes, contentType, signal })
+ if (!r.resp.ok) throw await failed('signed PUT', r)
+ await r.bytes()
+}
+
+/** Signed GET of `key` (whole object, or a `Range`). */
+export async function getObject(s: S3Settings, secrets: ProfileSecrets, key: string, range?: string, signal?: AbortSignal): Promise {
+ const r = await send(s, secrets, 'signed GET', 'GET', key, { range, signal })
+ if (!r.resp.ok) throw await failed('signed GET', r)
+ return r.bytes()
+}
+
+/** Signed HEAD: the object's size, or null when it does not exist. */
+export async function headObject(s: S3Settings, secrets: ProfileSecrets, key: string): Promise {
+ const r = await send(s, secrets, 'signed HEAD', 'HEAD', key)
+ if (!r.resp.ok && r.resp.status !== 404) throw await failed('signed HEAD', r)
+ r.discard()
+ if (r.resp.status === 404) return null
+ const len = Number(r.resp.headers.get('content-length'))
+ return Number.isFinite(len) ? len : 0
+}
+
+/** Signed DELETE of `key` (idempotent: a missing object is fine). */
+export async function deleteObject(s: S3Settings, secrets: ProfileSecrets, key: string): Promise {
+ const r = await send(s, secrets, 'delete', 'DELETE', key)
+ if (!r.resp.ok && r.resp.status !== 404) throw await failed('delete', r)
+ r.discard()
+}
diff --git a/forge-web/lib/storage/sigv4.test.ts b/forge-web/lib/storage/sigv4.test.ts
new file mode 100644
index 000000000..17824916f
--- /dev/null
+++ b/forge-web/lib/storage/sigv4.test.ts
@@ -0,0 +1,160 @@
+/**
+ * SigV4 parity with forge-core `backends/sigv4.rs`: the same AWS test-suite vectors and S3
+ * API reference examples, so the browser and the CLI sign byte for byte alike.
+ */
+
+import { describe, expect, it } from 'vitest'
+
+import {
+ EMPTY_PAYLOAD_SHA256,
+ amzDate,
+ canonicalHeaders,
+ canonicalQuery,
+ keyHasBadSegment,
+ sha256Hex,
+ signRequest,
+ uriEncode,
+ type SigningKeys,
+} from './sigv4'
+
+const SUITE: SigningKeys = { accessKeyId: 'AKIDEXAMPLE', secretAccessKey: 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY' }
+const SUITE_DATE = { date: '20150830', datetime: '20150830T123600Z' }
+
+async function suiteSign(
+ method: string,
+ path: string,
+ query: [string, string][] = [],
+ headers: [string, string][] = [],
+ sessionToken?: string,
+): Promise {
+ const out = await signRequest(
+ {
+ method,
+ host: 'example.amazonaws.com',
+ canonicalUri: uriEncode(path, true),
+ query,
+ headers,
+ payloadHash: EMPTY_PAYLOAD_SHA256,
+ region: 'us-east-1',
+ service: 'service',
+ contentSha256Header: false,
+ },
+ sessionToken ? { ...SUITE, sessionToken } : SUITE,
+ SUITE_DATE,
+ )
+ const auth = out.find(([k]) => k === 'authorization')?.[1] ?? ''
+ return auth.split('Signature=')[1] ?? ''
+}
+
+describe('SigV4 test suite (generic service)', () => {
+ it.each([
+ ['get-vanilla', 'GET', '/', [], [], '5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31'],
+ ['get-space-unnormalized', 'GET', '/example space/', [], [], '652487583200325589f1fba4c7e578f72c47cb61beeca81406b39ddec1366741'],
+ ['get-utf8', 'GET', '/ሴ', [], [], '8318018e0b0f223aa2bbf98705b62bb787dc9c0e678f255a891fd03141be5d85'],
+ [
+ 'get-unreserved',
+ 'GET',
+ '/-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz',
+ [],
+ [],
+ '07ef7494c76fa4850883e2b006601f940f8a34d404d0cfa977f52a65bbf5f24f',
+ ],
+ ['query-order-key-case', 'GET', '/', [['Param2', 'value2'], ['Param1', 'value1']], [], 'b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500'],
+ ['utf8-query-key', 'GET', '/', [['ሴ', 'bar']], [], '2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04'],
+ ['post-header-key-sort', 'POST', '/', [], [['My-Header1', 'value1']], 'c5410059b04c1ee005303aed430f6e6645f61f4dc9e1461ec8f8916fdf18852c'],
+ ] as const)('%s', async (_name, method, path, query, headers, want) => {
+ expect(await suiteSign(method, path, [...query] as [string, string][], [...headers] as [string, string][])).toBe(want)
+ })
+
+ it('signs the session token', async () => {
+ expect(await suiteSign('GET', '/', [], [], '6e86291e8372ff2a2260956d9b8aae1d763fbf315fa00fa31553b73ebf194267')).toBe(
+ '07ec1639c89043aa0e3e2de82b96708f198cceab042d4a97044c66dd9f74e7f8',
+ )
+ })
+})
+
+describe('S3 API reference examples', () => {
+ const S3: SigningKeys = { accessKeyId: 'AKIAIOSFODNN7EXAMPLE', secretAccessKey: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' }
+ const DATE = amzDate(Date.UTC(2013, 4, 24))
+
+ it('formats the UTC stamp', () => {
+ expect(DATE).toEqual({ date: '20130524', datetime: '20130524T000000Z' })
+ expect(amzDate(1_440_938_160_000).datetime).toBe('20150830T123600Z')
+ expect(amzDate(951_782_400_000).date).toBe('20000229')
+ })
+
+ it('GET object with a range', async () => {
+ const out = await signRequest(
+ {
+ method: 'GET',
+ host: 'examplebucket.s3.amazonaws.com',
+ canonicalUri: '/test.txt',
+ headers: [['range', 'bytes=0-9']],
+ payloadHash: EMPTY_PAYLOAD_SHA256,
+ region: 'us-east-1',
+ service: 's3',
+ contentSha256Header: true,
+ },
+ S3,
+ DATE,
+ )
+ expect(out.find(([k]) => k === 'authorization')?.[1]).toBe(
+ 'AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, ' +
+ 'SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, ' +
+ 'Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41',
+ )
+ })
+
+ it('PUT object whose key needs encoding', async () => {
+ const payload = await sha256Hex(new TextEncoder().encode('Welcome to Amazon S3.'))
+ expect(payload).toBe('44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072')
+ const uri = `/${uriEncode('test$file.text', true)}`
+ expect(uri).toBe('/test%24file.text')
+ const out = await signRequest(
+ {
+ method: 'PUT',
+ host: 'examplebucket.s3.amazonaws.com',
+ canonicalUri: uri,
+ headers: [
+ ['Date', 'Fri, 24 May 2013 00:00:00 GMT'],
+ ['x-amz-storage-class', 'REDUCED_REDUNDANCY'],
+ ],
+ payloadHash: payload,
+ region: 'us-east-1',
+ service: 's3',
+ contentSha256Header: true,
+ },
+ S3,
+ DATE,
+ )
+ expect(out.find(([k]) => k === 'authorization')?.[1]).toMatch(
+ /SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd$/,
+ )
+ })
+})
+
+describe('canonicalization helpers', () => {
+ it('encodes like forge-core', () => {
+ expect(uriEncode('a b+c=d&e', true)).toBe('a%20b%2Bc%3Dd%26e')
+ expect(uriEncode('dir/file', true)).toBe('dir/file')
+ expect(uriEncode('dir/file', false)).toBe('dir%2Ffile')
+ expect(uriEncode('ü', true)).toBe('%C3%BC')
+ expect(uriEncode('100%', true)).toBe('100%25')
+ })
+
+ it('trims, collapses and merges headers', () => {
+ expect(canonicalHeaders([['X-B', ' two words '], ['x-a', '1'], ['X-A', '2']])).toEqual({
+ canonical: 'x-a:1,2\nx-b:two words\n',
+ signed: 'x-a;x-b',
+ })
+ })
+
+ it('keeps = on an empty query value', () => {
+ expect(canonicalQuery([['b', ''], ['a', 'x y']])).toBe('a=x%20y&b=')
+ })
+
+ it('refuses keys an HTTP stack would normalize', () => {
+ expect(keyHasBadSegment('packs/abc.pack')).toBe(false)
+ for (const k of ['', 'a//b', './a', 'a/../b', 'a/']) expect(keyHasBadSegment(k)).toBe(true)
+ })
+})
diff --git a/forge-web/lib/storage/sigv4.ts b/forge-web/lib/storage/sigv4.ts
new file mode 100644
index 000000000..43f0d462f
--- /dev/null
+++ b/forge-web/lib/storage/sigv4.ts
@@ -0,0 +1,164 @@
+/**
+ * AWS Signature Version 4, for S3-compatible buckets (R2, B2, AWS S3, MinIO), on WebCrypto.
+ *
+ * A port of forge-core `backends/sigv4.rs` and pinned by the same vectors: the AWS SigV4 test
+ * suite and the worked examples in the S3 API reference (`sigv4.test.ts`). S3 canonicalization
+ * differs from generic SigV4 in one way that matters: the object path is percent-encoded ONCE,
+ * keeping `/`, and never normalized. A key with an empty, `.` or `..` segment is refused by the
+ * caller, because `fetch` would normalize it after signing and address another object.
+ *
+ * The secret access key is imported into a non-extractable HMAC `CryptoKey` for the first step
+ * of the key derivation and never formatted, logged or returned.
+ */
+
+/** SHA-256 of the empty string: the payload hash of a body-less request. */
+export const EMPTY_PAYLOAD_SHA256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
+
+const ALGORITHM = 'AWS4-HMAC-SHA256'
+const enc = new TextEncoder()
+
+/** The credentials a request is signed with. */
+export interface SigningKeys {
+ readonly accessKeyId: string
+ readonly secretAccessKey: string
+ readonly sessionToken?: string
+}
+
+/** A signing timestamp in the two forms SigV4 uses. */
+export interface AmzDate {
+ /** `YYYYMMDD` (the credential-scope date). */
+ readonly date: string
+ /** `YYYYMMDDTHHMMSSZ` (the `x-amz-date` value). */
+ readonly datetime: string
+}
+
+/** The UTC stamp of `ms` since the epoch. */
+export function amzDate(ms: number = Date.now()): AmzDate {
+ const iso = new Date(ms).toISOString() // 2013-05-24T00:00:00.000Z
+ const datetime = `${iso.slice(0, 4)}${iso.slice(5, 7)}${iso.slice(8, 10)}T${iso.slice(11, 13)}${iso.slice(14, 16)}${iso.slice(17, 19)}Z`
+ return { date: datetime.slice(0, 8), datetime }
+}
+
+/**
+ * RFC 3986 percent-encoding as SigV4 specifies it: every byte except `A-Z a-z 0-9 - . _ ~`
+ * becomes `%XX` (uppercase hex). With `keepSlash`, `/` stays (object-key paths).
+ */
+export function uriEncode(input: string, keepSlash: boolean): string {
+ let out = ''
+ for (const b of enc.encode(input)) {
+ const c = String.fromCharCode(b)
+ if (/[A-Za-z0-9\-._~]/.test(c) || (keepSlash && c === '/')) out += c
+ else out += `%${b.toString(16).toUpperCase().padStart(2, '0')}`
+ }
+ return out
+}
+
+/** Whether an object key has an empty, `.` or `..` segment (see the module doc). */
+export function keyHasBadSegment(key: string): boolean {
+ return key === '' || key.split('/').some((s) => s === '' || s === '.' || s === '..')
+}
+
+function hex(bytes: ArrayBuffer | Uint8Array): string {
+ const u = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes)
+ let s = ''
+ for (const b of u) s += b.toString(16).padStart(2, '0')
+ return s
+}
+
+function buf(b: Uint8Array): Uint8Array {
+ return new Uint8Array(b)
+}
+
+/** Lowercase-hex SHA-256 of `bytes`. */
+export async function sha256Hex(bytes: Uint8Array): Promise {
+ return hex(await crypto.subtle.digest('SHA-256', buf(bytes)))
+}
+
+async function hmac(key: Uint8Array | CryptoKey, data: string): Promise {
+ const k =
+ key instanceof Uint8Array
+ ? await crypto.subtle.importKey('raw', buf(key), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign'])
+ : key
+ return new Uint8Array(await crypto.subtle.sign('HMAC', k, enc.encode(data)))
+}
+
+/** The derived signing key: `HMAC(HMAC(HMAC(HMAC("AWS4"+secret, date), region), service), "aws4_request")`. */
+async function signingKey(secret: string, date: string, region: string, service: string): Promise {
+ const root = await crypto.subtle.importKey('raw', enc.encode(`AWS4${secret}`), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign'])
+ const kDate = await hmac(root, date)
+ const kRegion = await hmac(kDate, region)
+ const kService = await hmac(kRegion, service)
+ return hmac(kService, 'aws4_request')
+}
+
+/**
+ * The canonical query: each key and value encoded (slashes too), sorted by encoded key then
+ * value, joined with `&`. An empty value keeps its `=`.
+ */
+export function canonicalQuery(pairs: readonly (readonly [string, string])[]): string {
+ return pairs
+ .map(([k, v]) => [uriEncode(k, false), uriEncode(v, false)] as const)
+ .sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0))
+ .map(([k, v]) => `${k}=${v}`)
+ .join('&')
+}
+
+/**
+ * Canonical headers and the `SignedHeaders` list: names lowercased, values trimmed with inner
+ * whitespace runs collapsed, sorted by name (stable), duplicates joined by `,`.
+ */
+export function canonicalHeaders(headers: readonly (readonly [string, string])[]): { canonical: string; signed: string } {
+ const norm = headers
+ .map(([k, v]) => [k.trim().toLowerCase(), v.split(/\s+/).filter((p) => p !== '').join(' ')] as const)
+ .map((h, i) => ({ h, i }))
+ .sort((a, b) => (a.h[0] < b.h[0] ? -1 : a.h[0] > b.h[0] ? 1 : a.i - b.i))
+ .map((x) => x.h)
+ const merged: [string, string][] = []
+ for (const [k, v] of norm) {
+ const last = merged[merged.length - 1]
+ if (last && last[0] === k) last[1] = `${last[1]},${v}`
+ else merged.push([k, v])
+ }
+ return {
+ canonical: merged.map(([k, v]) => `${k}:${v}\n`).join(''),
+ signed: merged.map(([k]) => k).join(';'),
+ }
+}
+
+/** One request to sign. */
+export interface RequestToSign {
+ readonly method: string
+ /** The `Host` exactly as sent: `host`, or `host:port` for a non-default port. */
+ readonly host: string
+ /** The already-encoded path ({@link uriEncode} with `keepSlash`). */
+ readonly canonicalUri: string
+ readonly query?: readonly (readonly [string, string])[]
+ /** Extra headers to sign (`range`, `content-type`); the signer adds its own. */
+ readonly headers?: readonly (readonly [string, string])[]
+ /** Hex SHA-256 of the body, or {@link EMPTY_PAYLOAD_SHA256}. */
+ readonly payloadHash: string
+ readonly region: string
+ readonly service: string
+ /** Send and sign `x-amz-content-sha256` (S3 requires it; the generic suite does not). */
+ readonly contentSha256Header: boolean
+}
+
+/**
+ * Sign `req`: the headers to add to the outgoing request, `authorization` last. `host` is
+ * signed but not returned; `fetch` derives it from the URL, which must produce `req.host`.
+ */
+export async function signRequest(req: RequestToSign, keys: SigningKeys, when: AmzDate): Promise<[string, string][]> {
+ const added: [string, string][] = [['x-amz-date', when.datetime]]
+ if (req.contentSha256Header) added.push(['x-amz-content-sha256', req.payloadHash])
+ if (keys.sessionToken) added.push(['x-amz-security-token', keys.sessionToken])
+ const all: [string, string][] = [['host', req.host], ...(req.headers ?? []).map(([k, v]) => [k, v] as [string, string]), ...added]
+ const { canonical, signed } = canonicalHeaders(all)
+ const creq = [req.method, req.canonicalUri, canonicalQuery(req.query ?? []), canonical, signed, req.payloadHash].join('\n')
+ const scope = `${when.date}/${req.region}/${req.service}/aws4_request`
+ const sts = [ALGORITHM, when.datetime, scope, await sha256Hex(enc.encode(creq))].join('\n')
+ const key = await signingKey(keys.secretAccessKey, when.date, req.region, req.service)
+ const signature = hex(await hmac(key, sts))
+ key.fill(0)
+ added.push(['authorization', `${ALGORITHM} Credential=${keys.accessKeyId}/${scope}, SignedHeaders=${signed}, Signature=${signature}`])
+ return added
+}
diff --git a/forge-web/lib/storage/storage.test.ts b/forge-web/lib/storage/storage.test.ts
new file mode 100644
index 000000000..d0ea4d71d
--- /dev/null
+++ b/forge-web/lib/storage/storage.test.ts
@@ -0,0 +1,434 @@
+/**
+ * The storage layer, offline: profile validation, policies, URI ordering and budgets, chunk
+ * splitting (parity with forge-core `pack::split`), the CORS fix blocks, the config sealed in
+ * the vault, and the replication engine against in-memory S3 and kubo fakes.
+ */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { resetMemoryStores, idbEntries, idbGet, idbPut } from '../idb'
+import { lockVault, storeInVault, unlockWithPassphrase } from '../auth/vault'
+import { splitChunks, manifestUrisProblem } from '../repo/push'
+import { CHUNK_PAYLOAD_MAX, FIELD_MAX } from '../constants'
+import { isPrivateHost, isPublicHttpsUrl } from '../net'
+import { externalFetchUrls } from '../view/browse-source'
+import { corsFix } from './cors'
+import { cidV1RawLeaves } from './cid'
+import {
+ PLATFORM_PROFILE,
+ artifactKey,
+ choiceOf,
+ policyFor,
+ profileProblem,
+ profileSecretsSchema,
+ publishProblem,
+ type ProfilePublic,
+ type StorageProfile,
+} from './profiles'
+import {
+ EMPTY_STORAGE_CONFIG,
+ discardStorageConfig,
+ loadStorageConfig,
+ policyForRepo,
+ policyProblem,
+ saveStorageConfig,
+ withProfile,
+ withRenamedProfile,
+ withRepoPolicy,
+ withoutProfile,
+} from './store'
+import { ReplicationError, PlatformDeclinedError, fitManifestUris, orderUris, storeArtifact } from './upload'
+import { sha256Hex } from './sigv4'
+import type { WriteAuth } from '../sdk'
+import type { V2RepoRef } from '../repo/contract'
+import type { EvoSDK } from '@dashevo/evo-sdk'
+
+const ID = '9r27eDsuXEqoMNymW1A2MKFrpBhzSkepVKwXrGzq9dUD'
+const WIF = 'cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy'
+
+const S3: StorageProfile = {
+ name: 'r2-main',
+ settings: {
+ kind: 's3',
+ provider: 'r2',
+ endpoint: 'https://acct.r2.cloudflarestorage.com',
+ region: 'auto',
+ bucket: 'forge-byo',
+ pathStyle: true,
+ publicUrl: 'https://pub-9a1.r2.dev',
+ prefix: 'web',
+ },
+ secrets: { accessKeyId: 'AKIDEXAMPLE', secretAccessKey: 'r2-secret-value' },
+}
+
+const KUBO: StorageProfile = {
+ name: 'kubo',
+ settings: { kind: 'ipfs-kubo', provider: 'kubo', api: 'https://kubo.example', gateway: 'https://kubo-gw.example', publicGateway: '', pinningEndpoint: '' },
+ secrets: {},
+}
+
+const s3Of = (p: StorageProfile): Extract => p.settings as Extract
+
+describe('profiles', () => {
+ it('accepts a sound S3 profile and names what is wrong otherwise', () => {
+ expect(profileProblem(S3)).toBeNull()
+ const s3 = s3Of(S3)
+ const bad = (patch: Partial, secrets = S3.secrets): string | null => profileProblem({ ...S3, settings: { ...s3, ...patch }, secrets })
+ expect(bad({ endpoint: 'https://x.example/bucket' })).toMatch(/origin/)
+ expect(bad({ endpoint: 'https://user:pw@x.example' })).toMatch(/user name/)
+ expect(bad({ endpoint: 'http://minio.example.org' })).toMatch(/https/)
+ // The API endpoint may be plain http on this machine (the user's own MinIO); what is
+ // published may not (the public URL, checked by publishProblem).
+ expect(bad({ endpoint: 'http://127.0.0.1:9000' })).toBeNull()
+ expect(bad({ endpoint: 'http://192.168.1.4:9000' })).toMatch(/https/)
+ expect(bad({ bucket: 'Forge' })).toMatch(/lowercase/)
+ expect(bad({ prefix: 'a/../b' })).toMatch(/segment/)
+ expect(bad({ publicUrl: '' })).toMatch(/public URL/)
+ expect(bad({ publicUrl: 'http://pub.example' })).toMatch(/recorded on chain/)
+ expect(bad({ pathStyle: false, endpoint: 'https://1.2.3.4' })).toMatch(/path-style/)
+ expect(bad({}, {})).toMatch(/access key/)
+ expect(profileProblem({ ...S3, name: 'a,b' })).toMatch(/name/)
+ })
+
+ it('reserves the Platform profile name', () => {
+ expect(profileProblem({ ...S3, name: PLATFORM_PROFILE })).toMatch(/reserved/)
+ expect(profileProblem({ name: PLATFORM_PROFILE, settings: { kind: 'platform', provider: 'platform' }, secrets: {} })).toBeNull()
+ })
+
+ it('refuses secrets an HTTP header cannot carry, without echoing them', () => {
+ const problem = profileProblem({ ...S3, secrets: { ...S3.secrets, sessionToken: 'tok\nsecret-part' } })
+ expect(problem).toMatch(/session token/i)
+ expect(problem).not.toContain('secret-part')
+ expect(profileSecretsSchema.safeParse({ accessKeyId: 'ключ' }).success).toBe(false)
+ expect(profileSecretsSchema.safeParse({ accessKeyId: 'AKID', extra: 'x' }).success).toBe(false)
+ })
+
+ it('refuses a published address only its uploader can reach', () => {
+ expect(publishProblem(S3)).toBeNull()
+ for (const url of ['http://127.0.0.1:9000/forge-byo', 'https://192.168.1.4/b', 'https://10.0.0.8', 'https://minio.local/b', 'https://[::1]/b', 'https://[fd00::1]/b']) {
+ expect(publishProblem({ ...S3, settings: { ...s3Of(S3), publicUrl: url } }), url).toMatch(/other people cannot read/)
+ }
+ expect(publishProblem({ ...KUBO, settings: { ...KUBO.settings, publicGateway: 'http://localhost:8080' } as ProfilePublic })).toMatch(/other people/)
+ })
+
+ it('keys artifacts like the CLI', () => {
+ const s3 = s3Of(S3)
+ expect(artifactKey(s3, 'ab'.repeat(32))).toBe(`web/packs/${'ab'.repeat(32)}.pack`)
+ expect(artifactKey({ ...s3, prefix: '' }, 'cd')).toBe('packs/cd.pack')
+ expect(artifactKey({ ...s3, prefix: '/a/b//' }, 'cd')).toBe('a/b/packs/cd.pack')
+ })
+
+ it('maps replication choices to policies and back', () => {
+ expect(policyFor(['a', 'b'], 'one')).toEqual({ targets: ['a', 'b'], replicas: 1, platformFallback: false })
+ expect(policyFor(['a', 'b'], 'all')).toEqual({ targets: ['a', 'b'], replicas: 2, platformFallback: false })
+ expect(policyFor(['a'], 'fallback')).toEqual({ targets: ['a'], replicas: 1, platformFallback: true })
+ for (const c of ['one', 'all', 'fallback'] as const) expect(choiceOf(policyFor(['a', 'b'], c))).toBe(c)
+ })
+})
+
+describe('what readers may fetch', () => {
+ it('only public https hosts', () => {
+ for (const h of ['127.0.0.1', 'localhost', 'x.localhost', '10.1.2.3', '172.20.0.1', '192.168.0.9', '169.254.1.1', '100.64.0.1', '::1', 'fe80::1', 'fc00::5', '::ffff:127.0.0.1']) {
+ expect(isPrivateHost(h), h).toBe(true)
+ }
+ for (const h of ['pub-9a1.r2.dev', '8.8.8.8', '172.32.0.1', 'ipfs.io']) expect(isPrivateHost(h), h).toBe(false)
+ expect(isPublicHttpsUrl('http://pub.example/p')).toBe(false)
+ expect(isPublicHttpsUrl('https://pub.example/p')).toBe(true)
+ })
+
+ it('never makes a reader request a loopback or private URL a manifest records', () => {
+ expect(
+ externalFetchUrls(['http://127.0.0.1:9000/p', 'https://192.168.1.2/p', 'http://pub.example/p', 'https://pub.example/p', 'ipfs://bafyok', 'ipfs://bafy/../api'], ['https://ipfs.io']),
+ ).toEqual(['https://pub.example/p', 'https://ipfs.io/ipfs/bafyok'])
+ })
+})
+
+describe('the stored configuration', () => {
+ it('prunes a removed profile out of every policy', () => {
+ let c = withProfile(withProfile(EMPTY_STORAGE_CONFIG, S3), KUBO)
+ c = { ...c, defaultPolicy: policyFor(['r2-main', 'kubo'], 'all') }
+ c = withRepoPolicy(c, 'R1', policyFor(['kubo'], 'one'))
+ const after = withoutProfile(c, 'kubo')
+ expect(after.profiles.map((p) => p.name)).toEqual(['r2-main'])
+ expect(after.defaultPolicy).toEqual({ targets: ['r2-main'], replicas: 1, platformFallback: false })
+ expect(after.repoPolicies).toEqual({})
+ expect(policyForRepo(after, 'R1')).toEqual(after.defaultPolicy)
+ })
+
+ it('renames a profile inside every policy', () => {
+ let c = withProfile(withProfile(EMPTY_STORAGE_CONFIG, S3), KUBO)
+ c = { ...c, defaultPolicy: policyFor(['r2-main', 'kubo'], 'all'), lastTests: { kubo: { at: 1, ok: true } } }
+ c = withRepoPolicy(c, 'R1', policyFor(['kubo'], 'one'))
+ const after = withRenamedProfile(c, 'kubo', { ...KUBO, name: 'my-node' })
+ expect(after.profiles.map((p) => p.name)).toEqual(['my-node', 'r2-main'])
+ expect(after.defaultPolicy?.targets).toEqual(['r2-main', 'my-node'])
+ expect(after.repoPolicies['R1']?.targets).toEqual(['my-node'])
+ expect(after.lastTests).toEqual({})
+ })
+
+ it('refuses a policy naming unknown profiles or too many copies', () => {
+ const c = withProfile(EMPTY_STORAGE_CONFIG, S3)
+ expect(policyProblem(c, policyFor(['r2-main'], 'one'))).toBeNull()
+ expect(policyProblem(c, policyFor(['nope'], 'one'))).toMatch(/no storage profile/)
+ expect(policyProblem(c, { targets: ['r2-main'], replicas: 2, platformFallback: false })).toMatch(/copies/)
+ expect(policyProblem(c, { targets: [], replicas: 1, platformFallback: false })).toMatch(/at least one/)
+ })
+})
+
+describe('the configuration sealed in the vault', () => {
+ beforeEach(() => {
+ resetMemoryStores()
+ lockVault()
+ })
+
+ it('stores secrets encrypted, reads them back unlocked, and refuses when locked', async () => {
+ await storeInVault('devnet', { identityId: ID, keyId: 5, wif: WIF }, { passphrase: 'correct horse battery' })
+ const config = withRepoPolicy(withProfile(EMPTY_STORAGE_CONFIG, S3), 'R1', policyFor(['r2-main'], 'one'))
+ await saveStorageConfig('devnet', ID, config)
+ const dump = JSON.stringify(await idbEntries('vault'), (_k, v: unknown) => (v instanceof Uint8Array ? Array.from(v) : v))
+ expect(dump).not.toContain('r2-secret-value')
+ expect(dump).not.toContain('forge-byo')
+ expect(await loadStorageConfig('devnet', ID)).toEqual(config)
+ lockVault()
+ await expect(loadStorageConfig('devnet', ID)).rejects.toThrow(/unlock/)
+ await unlockWithPassphrase('devnet', ID, 'correct horse battery')
+ expect((await loadStorageConfig('devnet', ID)).profiles[0]?.secrets.secretAccessKey).toBe('r2-secret-value')
+ }, 60_000)
+
+ it('carries the storage settings across a key renewal made while unlocked', async () => {
+ await storeInVault('devnet', { identityId: ID, keyId: 5, wif: WIF }, { passphrase: 'correct horse battery' })
+ await saveStorageConfig('devnet', ID, withProfile(EMPTY_STORAGE_CONFIG, S3))
+ const outcome = await storeInVault('devnet', { identityId: ID, keyId: 6, wif: WIF }, { passphrase: 'a different passphrase' })
+ expect(outcome.storageSettingsDropped).toBe(false)
+ lockVault()
+ await unlockWithPassphrase('devnet', ID, 'a different passphrase')
+ expect((await loadStorageConfig('devnet', ID)).profiles.map((p) => p.name)).toEqual(['r2-main'])
+ }, 90_000)
+
+ it('says so when a renewal made while locked cannot carry them', async () => {
+ await storeInVault('devnet', { identityId: ID, keyId: 5, wif: WIF }, { passphrase: 'correct horse battery' })
+ await saveStorageConfig('devnet', ID, withProfile(EMPTY_STORAGE_CONFIG, S3))
+ lockVault()
+ const outcome = await storeInVault('devnet', { identityId: ID, keyId: 6, wif: WIF }, { passphrase: 'a different passphrase' })
+ expect(outcome.storageSettingsDropped).toBe(true)
+ expect(await loadStorageConfig('devnet', ID)).toEqual(EMPTY_STORAGE_CONFIG)
+ }, 90_000)
+
+ it('can discard settings sealed under a key it no longer has', async () => {
+ await storeInVault('devnet', { identityId: ID, keyId: 5, wif: WIF }, { passphrase: 'correct horse battery' })
+ await saveStorageConfig('devnet', ID, withProfile(EMPTY_STORAGE_CONFIG, S3))
+ const stale = await idbGet('vault', `vault-storage:devnet:${ID}`)
+ await storeInVault('devnet', { identityId: ID, keyId: 6, wif: WIF }, { passphrase: 'a different passphrase' })
+ await idbPut('vault', `vault-storage:devnet:${ID}`, stale)
+ await expect(loadStorageConfig('devnet', ID)).rejects.toThrow(/do not open/)
+ await discardStorageConfig('devnet', ID)
+ expect(await loadStorageConfig('devnet', ID)).toEqual(EMPTY_STORAGE_CONFIG)
+ }, 90_000)
+})
+
+describe('chunks and manifests', () => {
+ it('splits exactly as forge-core pack::split', () => {
+ expect(splitChunks(new Uint8Array(0))).toEqual([])
+ const bytes = new Uint8Array(CHUNK_PAYLOAD_MAX * 2 + FIELD_MAX + 7).map((_, i) => i % 251)
+ const chunks = splitChunks(bytes)
+ expect(chunks.map((c) => c.seq)).toEqual([0, 1, 2])
+ expect(chunks.map((c) => c.fields.map((f) => f.length))).toEqual([[FIELD_MAX, FIELD_MAX, FIELD_MAX], [FIELD_MAX, FIELD_MAX, FIELD_MAX], [FIELD_MAX, 7]])
+ const joined = new Uint8Array(chunks.flatMap((c) => c.fields.flatMap((f) => Array.from(f))))
+ expect(joined).toEqual(bytes)
+ })
+
+ it('orders URIs platform, then https, then the rest, and fits the budget', () => {
+ expect(orderUris([['https://a/p', 's3://b/p'], ['ipfs://bafy', 'https://gw/ipfs/bafy'], ['platform://C/R/O/h']])).toEqual([
+ 'platform://C/R/O/h',
+ 'https://a/p',
+ 'https://gw/ipfs/bafy',
+ 's3://b/p',
+ 'ipfs://bafy',
+ ])
+ const many = Array.from({ length: 5 }, (_, i) => [`https://h${i}.example/p`, `s3://b${i}/p`])
+ expect(fitManifestUris(orderUris(many))).toHaveLength(5)
+ expect(() => fitManifestUris([`https://x.example/${'a'.repeat(400)}`])).toThrow(/do not fit/)
+ expect(manifestUrisProblem([])).toMatch(/no confirmed copy/)
+ })
+
+ it('never records an address others cannot read', () => {
+ expect(() => fitManifestUris(['http://127.0.0.1:9000/b/p', 's3://b/p'])).toThrow(/only public https/)
+ expect(() => fitManifestUris(['https://192.168.1.2/p'])).toThrow(/only public https/)
+ expect(() => fitManifestUris(['http://pub.example/p'])).toThrow(/only public https/)
+ })
+})
+
+describe('CORS fix blocks', () => {
+ it('are valid JSON naming the bucket, Range for reads and PUT for this origin', () => {
+ for (const p of ['r2', 'aws', 'b2'] as const) {
+ const fix = corsFix(p, 'my-bucket', 'https://forge.dashhq.org')
+ expect(fix.where + fix.text).toContain('my-bucket')
+ const text = JSON.stringify(JSON.parse(fix.text) as unknown).toLowerCase()
+ expect(text).toContain('range')
+ expect(text).toMatch(/put/)
+ expect(text).toContain('https://forge.dashhq.org')
+ expect(text).toContain('x-amz-content-sha256')
+ }
+ })
+
+ it('keeps MinIO reads open to every origin', () => {
+ expect(corsFix('minio', 'b', 'https://forge.dashhq.org').text).toContain('cors_allow_origin="*"')
+ })
+
+ it('gives kubo a restricted token and merges the origin into the existing list', () => {
+ const { where, text } = corsFix('kubo', '', 'https://forge.dashhq.org')
+ expect(where).toMatch(/admin interface/)
+ expect(text).toContain('API.Authorizations.dash-forge')
+ for (const path of ['/api/v0/add', '/api/v0/pin/ls', '/api/v0/pin/rm', '/api/v0/id', '/api/v0/version']) expect(text).toContain(path)
+ expect(text).toMatch(/\(ipfs config API\.HTTPHeaders\.Access-Control-Allow-Origin 2>\/dev\/null \|\| echo null\) \| jq -c '\(\. \/\/ \[\]\) \+ \["https:\/\/forge\.dashhq\.org"\] \| unique'/)
+ const commands = text.split('\n').filter((l) => !l.startsWith('#'))
+ expect(commands.join('\n')).not.toMatch(/Access-Control-Allow-Origin '\["https:\/\/forge\.dashhq\.org"\]'/)
+ // Any Authorizations entry locks the RPC API for everyone else: an owner token comes first.
+ expect(text).toMatch(/API\.Authorizations\.owner .*\\"AllowedPaths\\": \[\\"\/api\/v0\\"\]/)
+ expect(text.indexOf('API.Authorizations.owner')).toBeLessThan(text.indexOf('API.Authorizations.dash-forge'))
+ })
+})
+
+// ---------------------------------------------------------------------------
+// The replication engine against in-memory S3 and kubo
+// ---------------------------------------------------------------------------
+
+const REPO: V2RepoRef = { kind: 'v2', forge: { core: 'CORE', collab: 'COLLAB', group: 'G' }, repoId: 'REPO', ownerId: ID, name: 'r', visibility: 'public' }
+const AUTH: WriteAuth = { identityId: ID, network: 'devnet', getSigningKeyWif: () => '' }
+const SDK = {} as EvoSDK
+
+/** Serve `obj` for a (possibly ranged) GET. */
+function serve(obj: Uint8Array, init?: RequestInit): Response {
+ const range = new Headers(init?.headers).get('range')
+ const m = range ? /^bytes=(\d+)-(\d+)$/.exec(range) : null
+ if (!m) return new Response(new Uint8Array(obj), { status: 200 })
+ const [start, end] = [Number(m[1]), Number(m[2])]
+ return new Response(obj.slice(start, end + 1), { status: 206, headers: { 'content-range': `bytes ${start}-${end}/${obj.length}` } })
+}
+
+function fakeNetwork(opts: { s3Down?: boolean; corruptOnce?: boolean; publicDenied?: boolean } = {}) {
+ const objects = new Map()
+ let corrupt = opts.corruptOnce === true
+ const puts: string[] = []
+ const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit): Promise => {
+ const url = new URL(String(input))
+ const method = init?.method ?? 'GET'
+ if (url.host === 'acct.r2.cloudflarestorage.com') {
+ if (opts.s3Down) throw new TypeError('Failed to fetch')
+ const key = url.pathname.replace(/^\/forge-byo\//, '')
+ if (method === 'PUT') {
+ const body = new Uint8Array(init?.body as ArrayBuffer | Uint8Array)
+ puts.push(key)
+ objects.set(key, new Uint8Array(corrupt ? body.map((b) => b ^ 1) : body))
+ corrupt = false
+ return new Response(null, { status: 200 })
+ }
+ const obj = objects.get(key)
+ if (!obj) return new Response(null, { status: 404 })
+ if (method === 'HEAD') return new Response(null, { status: 200, headers: { 'content-length': String(obj.length) } })
+ return serve(obj, init)
+ }
+ if (url.host === 'pub-9a1.r2.dev') {
+ if (opts.publicDenied) return new Response('AccessDenied', { status: 403 })
+ const obj = objects.get(url.pathname.slice(1))
+ return obj ? serve(obj, init) : new Response(null, { status: 404 })
+ }
+ if (url.host === 'kubo.example') {
+ if (url.pathname.endsWith('/add')) {
+ const file = (init?.body as FormData).get('file') as Blob
+ const bytes = new Uint8Array(await file.arrayBuffer())
+ const cid = cidV1RawLeaves(bytes)
+ objects.set(`ipfs:${cid}`, bytes)
+ return new Response(JSON.stringify({ Hash: cid }))
+ }
+ if (url.pathname.endsWith('/pin/ls')) {
+ const cid = url.searchParams.get('arg') ?? ''
+ return objects.has(`ipfs:${cid}`) ? new Response(JSON.stringify({ Keys: { [cid]: { Type: 'recursive' } } })) : new Response('{"Message":"not pinned"}', { status: 500 })
+ }
+ return new Response('{}')
+ }
+ if (url.host === 'kubo-gw.example' || url.host === 'ipfs.example') {
+ const obj = objects.get(`ipfs:${url.pathname.split('/').pop() ?? ''}`)
+ return obj ? serve(obj, init) : new Response(null, { status: 404 })
+ }
+ throw new Error(`unexpected ${url}`)
+ })
+ return { fetchMock, objects, puts }
+}
+
+describe('storeArtifact', () => {
+ const bytes = new TextEncoder().encode('PACK a small pack')
+ afterEach(() => vi.unstubAllGlobals())
+
+ it('stores on S3, verifies through the API and the public URL, and records the public URL first', async () => {
+ const net = fakeNetwork()
+ vi.stubGlobal('fetch', net.fetchMock)
+ const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main'], 'one'), profiles: [S3], confirmPlatform: async () => false })
+ const hash = await sha256Hex(bytes)
+ expect(stored).toMatchObject({ packHash: hash, storage: 1, chunkCount: 0, confirmed: ['r2-main'], failures: [] })
+ expect(stored.uris).toEqual([`https://pub-9a1.r2.dev/web/packs/${hash}.pack`, `s3://forge-byo/web/packs/${hash}.pack`])
+ const put = net.fetchMock.mock.calls.find(([, init]) => init?.method === 'PUT')
+ expect(new Headers(put?.[1]?.headers).get('authorization')).toMatch(/^AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE\//)
+ expect(new Headers(put?.[1]?.headers).get('x-amz-content-sha256')).toBe(hash)
+ // The public URL was read anonymously (no Authorization) before the copy counted.
+ const pub = net.fetchMock.mock.calls.find(([u]) => String(u).startsWith('https://pub-9a1.r2.dev/'))
+ expect(pub).toBeDefined()
+ expect(new Headers(pub?.[1]?.headers).get('authorization')).toBeNull()
+ })
+
+ it('does not count a copy whose public URL others cannot read', async () => {
+ vi.stubGlobal('fetch', fakeNetwork({ publicDenied: true }).fetchMock)
+ await expect(storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main'], 'one'), profiles: [S3], confirmPlatform: async () => false })).rejects.toThrow(
+ /public URL \(pub-9a1\.r2\.dev\) answered HTTP 403/,
+ )
+ })
+
+ it('refuses to upload to a profile whose public address only this machine can read', async () => {
+ const net = fakeNetwork()
+ vi.stubGlobal('fetch', net.fetchMock)
+ const local = { ...S3, settings: { ...s3Of(S3), publicUrl: 'https://192.168.1.20/forge-byo' } }
+ await expect(storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main'], 'one'), profiles: [local], confirmPlatform: async () => false })).rejects.toThrow(/other people cannot read/)
+ expect(net.puts).toEqual([])
+ })
+
+ it('re-uploads once when the stored copy does not verify', async () => {
+ const net = fakeNetwork({ corruptOnce: true })
+ vi.stubGlobal('fetch', net.fetchMock)
+ const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main'], 'one'), profiles: [S3], confirmPlatform: async () => false })
+ expect(stored.confirmed).toEqual(['r2-main'])
+ expect(net.puts).toHaveLength(2)
+ })
+
+ it('fails the policy (writing nothing to Platform) when too few targets confirm', async () => {
+ vi.stubGlobal('fetch', fakeNetwork({ s3Down: true }).fetchMock)
+ const confirm = vi.fn(async () => true)
+ const run = storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['r2-main', 'kubo'], 'all'), profiles: [S3, KUBO], confirmPlatform: confirm })
+ await expect(run).rejects.toBeInstanceOf(ReplicationError)
+ await expect(run).rejects.toThrow(/1 of 2 required .*r2-main: signed PUT.*Nothing was written to Platform/)
+ expect(confirm).not.toHaveBeenCalled()
+ })
+
+ it('asks, with the price, before a policy that names Platform writes any chunk', async () => {
+ vi.stubGlobal('fetch', fakeNetwork().fetchMock)
+ const platform: StorageProfile = { name: PLATFORM_PROFILE, settings: { kind: 'platform', provider: 'platform' }, secrets: {} }
+ const confirm = vi.fn(async () => false)
+ const run = storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor([PLATFORM_PROFILE], 'one'), profiles: [platform], confirmPlatform: confirm })
+ await expect(run).rejects.toBeInstanceOf(ReplicationError)
+ await expect(run).rejects.toThrow(/declined.*Nothing was written to Platform/)
+ expect(confirm).toHaveBeenCalledWith(expect.objectContaining({ bytes: bytes.length, estimateCredits: expect.any(Number), reason: expect.stringMatching(/includes Dash Platform/) }))
+ })
+
+ it('verifies the kubo CID, the node gateway and the public gateway', async () => {
+ vi.stubGlobal('fetch', fakeNetwork().fetchMock)
+ const withPublic = { ...KUBO, settings: { ...KUBO.settings, publicGateway: 'https://ipfs.example' } as ProfilePublic }
+ const stored = await storeArtifact(SDK, AUTH, REPO, bytes, { policy: policyFor(['kubo'], 'one'), profiles: [withPublic], confirmPlatform: async () => false })
+ const cid = cidV1RawLeaves(bytes)
+ expect(stored.uris).toEqual([`https://ipfs.example/ipfs/${cid}`, `ipfs://${cid}`])
+ })
+
+ it('asks before storing on Platform when nothing is configured, and stops when declined', async () => {
+ const confirm = vi.fn(async () => false)
+ await expect(storeArtifact(SDK, AUTH, REPO, bytes, { policy: null, profiles: [], confirmPlatform: confirm })).rejects.toBeInstanceOf(PlatformDeclinedError)
+ expect(confirm).toHaveBeenCalledWith(expect.objectContaining({ bytes: bytes.length, estimateCredits: expect.any(Number) }))
+ })
+})
diff --git a/forge-web/lib/storage/store.ts b/forge-web/lib/storage/store.ts
new file mode 100644
index 000000000..bdce94349
--- /dev/null
+++ b/forge-web/lib/storage/store.ts
@@ -0,0 +1,155 @@
+/**
+ * The browser's storage configuration: named profiles, the default browser-push policy, and
+ * per-repo overrides. All of it lives in ONE blob sealed in the vault (`lib/auth/vault.ts`
+ * `writeStorageBlob`): the profiles' secrets must never touch localStorage or plaintext
+ * IndexedDB, and keeping the non-secret parts next to them means there is exactly one place
+ * to read, one to forget, and nothing to reconcile. Reading needs an unlocked session, which
+ * every push needs anyway.
+ *
+ * What is stored is parsed with zod on the way out: a tampered or foreign blob fails to
+ * decrypt (AES-GCM with the network and identity as additional data) and, if it did not,
+ * would still not reach the uploader unvalidated.
+ */
+
+import { z } from 'zod'
+
+import type { Network } from '../constants'
+import { discardStorageBlob, readStorageBlob, writeStorageBlob } from '../auth/vault'
+import {
+ policySchema,
+ renameInPolicy,
+ profilePublicSchema,
+ profileSecretsSchema,
+ type StoragePolicy,
+ type StorageProfile,
+} from './profiles'
+
+/** Everything this browser knows about the user's storage. */
+export interface StorageConfig {
+ readonly version: 1
+ readonly profiles: readonly StorageProfile[]
+ /** The policy browser pushes use when a repo has no override. */
+ readonly defaultPolicy: StoragePolicy | null
+ /** repoId → the policy for browser pushes to that repo. */
+ readonly repoPolicies: Readonly>
+ /** Profile name → the last browser test: when, and whether every row passed. */
+ readonly lastTests: Readonly>
+}
+
+export const EMPTY_STORAGE_CONFIG: StorageConfig = {
+ version: 1,
+ profiles: [],
+ defaultPolicy: null,
+ repoPolicies: {},
+ lastTests: {},
+}
+
+const configSchema = z.object({
+ version: z.literal(1),
+ profiles: z
+ .array(z.object({ name: z.string().max(64), settings: profilePublicSchema, secrets: profileSecretsSchema }))
+ .max(32),
+ defaultPolicy: policySchema.nullable(),
+ repoPolicies: z.record(z.string().max(64), policySchema),
+ lastTests: z.record(z.string().max(64), z.object({ at: z.number(), ok: z.boolean() })),
+})
+
+/** The stored configuration (empty when none is stored). Needs the vault unlocked. */
+export async function loadStorageConfig(network: Network, identityId: string): Promise {
+ const raw = await readStorageBlob(network, identityId)
+ if (raw === null) return EMPTY_STORAGE_CONFIG
+ const parsed = configSchema.safeParse(raw)
+ if (!parsed.success) throw new Error('the stored storage settings are not in a shape this app understands')
+ return parsed.data as StorageConfig
+}
+
+/** Seal `config` in the vault. Needs the vault unlocked. */
+export async function saveStorageConfig(network: Network, identityId: string, config: StorageConfig): Promise {
+ const parsed = configSchema.parse(config)
+ await writeStorageBlob(network, identityId, parsed)
+}
+
+/**
+ * Delete stored storage settings this key cannot open (sealed under an earlier key, or in a
+ * shape this app does not understand). The profiles in them are lost: add them again.
+ */
+export function discardStorageConfig(network: Network, identityId: string): Promise {
+ return discardStorageBlob(network, identityId)
+}
+
+/**
+ * `config` with the profile `from` replaced by `profile`: every policy that named it follows
+ * a rename, as does its last test result.
+ */
+export function withRenamedProfile(config: StorageConfig, from: string, profile: StorageProfile): StorageConfig {
+ const to = profile.name
+ const repoPolicies: Record = {}
+ for (const [repo, p] of Object.entries(config.repoPolicies)) repoPolicies[repo] = renameInPolicy(p, from, to)
+ const lastTests = { ...config.lastTests }
+ delete lastTests[from]
+ return withProfile(
+ {
+ ...config,
+ profiles: config.profiles.filter((p) => p.name !== from),
+ defaultPolicy: config.defaultPolicy ? renameInPolicy(config.defaultPolicy, from, to) : null,
+ repoPolicies,
+ lastTests,
+ },
+ profile,
+ )
+}
+
+/** `config` with `profile` added or replaced (by name). */
+export function withProfile(config: StorageConfig, profile: StorageProfile): StorageConfig {
+ const profiles = [...config.profiles.filter((p) => p.name !== profile.name), profile].sort((a, b) => a.name.localeCompare(b.name))
+ return { ...config, profiles }
+}
+
+/**
+ * `config` without the profile `name`: it also leaves every policy (a policy naming a profile
+ * that no longer exists would fail every push). A policy left with no targets is removed.
+ */
+export function withoutProfile(config: StorageConfig, name: string): StorageConfig {
+ const prune = (p: StoragePolicy | null): StoragePolicy | null => {
+ if (p === null) return null
+ const targets = p.targets.filter((t) => t !== name)
+ if (targets.length === 0) return null
+ return { ...p, targets, replicas: Math.min(p.replicas, targets.length) }
+ }
+ const repoPolicies: Record = {}
+ for (const [repo, p] of Object.entries(config.repoPolicies)) {
+ const next = prune(p)
+ if (next) repoPolicies[repo] = next
+ }
+ const lastTests = { ...config.lastTests }
+ delete lastTests[name]
+ return {
+ ...config,
+ profiles: config.profiles.filter((p) => p.name !== name),
+ defaultPolicy: prune(config.defaultPolicy),
+ repoPolicies,
+ lastTests,
+ }
+}
+
+/** The policy browser pushes to `repoId` use: its override, else the default, else none. */
+export function policyForRepo(config: StorageConfig, repoId: string): StoragePolicy | null {
+ return config.repoPolicies[repoId] ?? config.defaultPolicy
+}
+
+/** `config` with `repoId`'s override set (or cleared with null). */
+export function withRepoPolicy(config: StorageConfig, repoId: string, policy: StoragePolicy | null): StorageConfig {
+ const repoPolicies = { ...config.repoPolicies }
+ if (policy === null) delete repoPolicies[repoId]
+ else repoPolicies[repoId] = policy
+ return { ...config, repoPolicies }
+}
+
+/** Why a policy cannot be used with `config`'s profiles, or null. */
+export function policyProblem(config: StorageConfig, policy: StoragePolicy): string | null {
+ if (policy.targets.length === 0) return 'choose at least one place to store packs'
+ const missing = policy.targets.filter((t) => !config.profiles.some((p) => p.name === t))
+ if (missing.length > 0) return `no storage profile named ${missing.join(', ')}`
+ if (policy.replicas > policy.targets.length) return `${policy.replicas} copies are required but only ${policy.targets.length} places are chosen`
+ return null
+}
diff --git a/forge-web/lib/storage/upload.live.test.ts b/forge-web/lib/storage/upload.live.test.ts
new file mode 100644
index 000000000..8025128e2
--- /dev/null
+++ b/forge-web/lib/storage/upload.live.test.ts
@@ -0,0 +1,106 @@
+/**
+ * Live browser-upload path — SKIPPED by default (devnet writes + a local MinIO).
+ *
+ * docker compose -f infra/docker-compose.yml up -d minio minio-init
+ * FORGE_LIVE=1 FORGE_LIVE_PUBLIC_URL=https://…/forge-byo NEXT_PUBLIC_NETWORK=devnet NEXT_PUBLIC_DEVNET_NAME=moutai \
+ * pnpm exec vitest run lib/storage/upload.live.test.ts
+ *
+ * As the moutai MAINTAINER (a maintainer of its own `forge-v2-empty`, the fixture no push has
+ * touched): store a small artifact in the `forge-byo` bucket with SigV4 (the same code the
+ * browser runs, on Node's WebCrypto + fetch), verified by re-read; write its `packManifest`
+ * (kind 2, a flatIndex slot nothing reads for browsing); then read the manifest back from
+ * Platform and fetch + hash the bytes through the web reader (`loadArtifactBytesProgress`),
+ * as any browser would. A rerun is idempotent (content-addressed key, the signer's own
+ * manifest slot). Spend: one manifest document.
+ */
+
+import { readFileSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+
+import { describe, expect, it } from 'vitest'
+
+import { DEFAULT_NETWORK, NETWORKS } from '../constants'
+import { isPublicHttpsUrl } from '../net'
+import { evoSdkService, type WriteAuth } from '../sdk'
+import { parseIdentityFileText } from '../auth'
+import { readV2PackCopies } from '../repo'
+import { loadArtifactBytesProgress } from '../view/browse-source'
+import { retryWhileMissing } from '../view/retry'
+import type { V2RepoRef } from '../repo/contract'
+import { storeAndRecordPack, policyFor, type StorageProfile } from './index'
+import { sha256Hex } from './sigv4'
+
+/**
+ * The bucket's PUBLIC https address: a manifest records only addresses anyone can read, so a
+ * bare local MinIO does not qualify. Expose it for the run, e.g.
+ * `cloudflared tunnel --url http://127.0.0.1:9000` → FORGE_LIVE_PUBLIC_URL=https://.trycloudflare.com/forge-byo
+ */
+const PUBLIC_URL = (process.env['FORGE_LIVE_PUBLIC_URL'] ?? '').replace(/\/+$/, '')
+const LIVE = process.env['FORGE_LIVE'] === '1' && DEFAULT_NETWORK === 'devnet' && isPublicHttpsUrl(PUBLIC_URL)
+const MAINTAINER_FILE = join(homedir(), '.config/dash-forge/test-identities/devnet-moutai/MAINTAINER.identity.json')
+const EMPTY_REPO = 'F9puk5NBQyySFbgj9yUfubdXAVv1YV7Zk2KDvfyHzxLU'
+
+const MINIO: StorageProfile = {
+ name: 'minio',
+ settings: {
+ kind: 's3',
+ provider: 'minio',
+ endpoint: 'http://127.0.0.1:9000',
+ region: 'us-east-1',
+ bucket: 'forge-byo',
+ pathStyle: true,
+ publicUrl: PUBLIC_URL,
+ prefix: 'web-live',
+ },
+ secrets: { accessKeyId: 'minioadmin', secretAccessKey: 'minioadmin' },
+}
+
+describe.skipIf(!LIVE)('live browser upload to MinIO + manifest on moutai', () => {
+ it(
+ 'stores, verifies, records, and reads back through the web reader',
+ async () => {
+ const parsed = parseIdentityFileText(readFileSync(MAINTAINER_FILE, 'utf8'))
+ const auth: WriteAuth = { identityId: parsed.identityId, network: 'devnet', getSigningKeyWif: () => parsed.signingKeyWif }
+ const v2 = NETWORKS.devnet.v2
+ if (!v2) throw new Error('no forge-v2 deployment on this devnet')
+ await evoSdkService.initialize({ network: 'devnet', contractIds: [v2.core, v2.collab], timeoutMs: 20000 })
+ const sdk = evoSdkService.getSdk()
+ const repo: V2RepoRef = { kind: 'v2', forge: v2, repoId: EMPTY_REPO, ownerId: parsed.identityId, name: 'forge-v2-empty', visibility: 'public' }
+
+ // One artifact per public URL: the signer's manifest slot for a pack is permanent, so a
+ // rerun with the same bytes and URL finds its own manifest and writes nothing.
+ const bytes = new TextEncoder().encode(`forge-web live upload artifact ${EMPTY_REPO} ${PUBLIC_URL}\n`)
+ const hash = await sha256Hex(bytes)
+ const events: string[] = []
+ const { stored, manifest } = await storeAndRecordPack(
+ sdk,
+ auth,
+ repo,
+ bytes,
+ { kind: 2, objectCount: 0 },
+ {
+ policy: policyFor(['minio'], 'one'),
+ profiles: [MINIO],
+ confirmPlatform: async () => false,
+ onStep: (e) => events.push(`${e.target}:${e.phase}`),
+ intent: `live-upload:${hash}`,
+ },
+ )
+ // eslint-disable-next-line no-console
+ console.log('stored', { uris: stored.uris, manifest: manifest.documentId, cost: manifest.cost.dash, events })
+ expect(stored.storage).toBe(1)
+ expect(stored.uris[0]).toBe(`${PUBLIC_URL}/web-live/packs/${hash}.pack`)
+ expect(stored.uris[1]).toBe(`s3://forge-byo/web-live/packs/${hash}.pack`)
+ expect(events).toEqual(['minio:start', 'minio:done'])
+
+ // The node answering the read may be a block behind the one that confirmed the write.
+ const copy = await retryWhileMissing(() => readV2PackCopies(sdk, repo, hash, 2), 8)
+ expect(copy, 'the manifest should read back from Platform').not.toBeNull()
+ expect(copy?.uris).toEqual(stored.uris)
+ const got = await loadArtifactBytesProgress(sdk, repo, copy!)
+ expect(await sha256Hex(got)).toBe(hash)
+ },
+ 180_000,
+ )
+})
diff --git a/forge-web/lib/storage/upload.ts b/forge-web/lib/storage/upload.ts
new file mode 100644
index 000000000..e35b13591
--- /dev/null
+++ b/forge-web/lib/storage/upload.ts
@@ -0,0 +1,336 @@
+/**
+ * Browser uploads of pack artifacts under the user's storage policy: the web counterpart of
+ * forge-core `storage/targets.rs` (`replicate`, `ExternalTarget`) and `StoredArtifact`.
+ *
+ * Order of events, the same guarantee the CLI gives: the bytes are stored AND verified on at
+ * least `replicas` targets before this resolves, and only then may the caller write the
+ * `packManifest` (and after it, refs). Nothing is written to Platform without the user first
+ * agreeing to its price ({@link StoreOptions.confirmPlatform}) — asked once, before the first
+ * chunk, whether Platform is a target of the policy, the fallback, or the only place.
+ *
+ * A copy counts as confirmed only when the URLs it records are readable by anyone, as in the
+ * CLI (`ExternalTarget::store` re-reads through the first recorded URI):
+ * - S3: the signed API re-read (whole object up to 16 MiB re-hashed; larger: size plus
+ * byte-exact head and tail windows), then an anonymous ranged read of the head and tail
+ * through the public URL — the one readers use;
+ * - IPFS: kubo's CID must equal the local derivation and be pinned, then a re-read through the
+ * node's gateway when one is configured, then an anonymous read through the public gateway
+ * when one is recorded.
+ * Every URL recorded must be a public https URL (`lib/net.ts`); a profile whose public address
+ * is not is refused before any upload.
+ */
+
+import type { EvoSDK } from '@dashevo/evo-sdk'
+
+import type { WriteAuth } from '../sdk'
+import { estimateChunkCredits } from '../sdk/cost'
+import { isRecordableUri } from '../net'
+import type { V2RepoRef } from '../repo/contract'
+import { manifestUrisProblem, putPlatformChunks } from '../repo/push'
+import { addVerified, gatewayUrl, remotePin } from './ipfs'
+import { artifactKey, profileProblem, publishProblem, type StoragePolicy, type StorageProfile } from './profiles'
+import { getObject, getPublic, headObject, putObject, publicObjectUrl, s3Uri, type S3Settings } from './s3'
+import { sha256Hex } from './sigv4'
+import { bytesEqual, errText, timedFetch } from './util'
+
+/** Artifacts up to this size are verified by a full re-download and SHA-256. */
+export const FULL_VERIFY_MAX = 16 * 1024 * 1024
+const EDGE_WINDOW = 64 * 1024
+
+/** A target that did not confirm. */
+export interface TargetFailure {
+ readonly target: string
+ readonly reason: string
+}
+
+/** Progress of one target. */
+export type UploadEvent =
+ | { readonly target: string; readonly phase: 'start' }
+ | { readonly target: string; readonly phase: 'progress'; readonly done: number; readonly total: number }
+ | { readonly target: string; readonly phase: 'done'; readonly uris: readonly string[] }
+ | { readonly target: string; readonly phase: 'failed'; readonly reason: string }
+
+/** Where an artifact now lives: what its `packManifest` records. */
+export interface StoredArtifact {
+ readonly packHash: string
+ readonly sizeBytes: number
+ /** 0 when a Platform copy exists (chunks), else 1. */
+ readonly storage: 0 | 1
+ readonly chunkCount: number
+ /** `platform://` first, then public https URLs, then the rest (`ipfs://`, `s3://`). */
+ readonly uris: readonly string[]
+ /** Targets that confirmed, in policy order. */
+ readonly confirmed: readonly string[]
+ readonly failures: readonly TargetFailure[]
+}
+
+/** The price of putting `bytes` on Platform, asked before paying it. */
+export interface PlatformQuestion {
+ readonly bytes: number
+ readonly estimateCredits: number
+ /** Why Platform is being used: a target of the policy, the fallback, or nothing configured. */
+ readonly reason: string
+}
+
+/** Fewer than the required targets confirmed. */
+export class ReplicationError extends Error {
+ constructor(
+ readonly required: number,
+ readonly confirmed: readonly string[],
+ readonly failures: readonly TargetFailure[],
+ /** Whether any Platform chunks were written before the policy failed (they stay: chunks are permanent). */
+ readonly platformWritten: boolean,
+ ) {
+ super(
+ `storage policy not met: ${confirmed.length} of ${required} required target(s) confirmed` +
+ failures.map((f) => `; ${f.target}: ${f.reason}`).join('') +
+ (platformWritten
+ ? '. Some pack chunks were written to Platform before the push stopped; they are kept, and a retry does not pay for them again. No manifest or ref was written.'
+ : '. Nothing was written to Platform.'),
+ )
+ this.name = 'ReplicationError'
+ }
+}
+
+/** The user declined to store on Platform. */
+export class PlatformDeclinedError extends Error {
+ constructor() {
+ super('Nothing was stored: storing on Platform was declined. Configure storage in Settings → Storage and try again.')
+ this.name = 'PlatformDeclinedError'
+ }
+}
+
+/** The byte windows compared at both ends of an artifact (the whole of a small one). */
+function edgeWindows(size: number): (readonly [number, number])[] {
+ if (size <= 2 * EDGE_WINDOW) return [[0, size]]
+ return [
+ [0, EDGE_WINDOW],
+ [size - EDGE_WINDOW, size],
+ ]
+}
+
+/** Re-read an S3 copy through the signed API and check it is exactly `bytes`. */
+async function verifyS3(s: S3Settings, p: StorageProfile, key: string, bytes: Uint8Array, hashHex: string): Promise {
+ if (bytes.length <= FULL_VERIFY_MAX) {
+ const got = await getObject(s, p.secrets, key)
+ if ((await sha256Hex(got)) !== hashHex) throw new Error(`re-read of ${key} returned ${got.length} bytes that do not hash to the artifact`)
+ return
+ }
+ const size = await headObject(s, p.secrets, key)
+ if (size !== bytes.length) throw new Error(`re-read of ${key}: object missing or wrong size (${size} ≠ ${bytes.length})`)
+ for (const [start, end] of edgeWindows(bytes.length)) {
+ const got = await getObject(s, p.secrets, key, `bytes=${start}-${end - 1}`)
+ if (!bytesEqual(got, bytes.subarray(start, end))) throw new Error(`re-read of ${key} bytes ${start}..${end} differ from the artifact`)
+ }
+}
+
+/** Read the head and tail of a copy anonymously, the way every reader will. */
+async function verifyPublic(label: string, read: (range: string) => Promise<{ status: number; bytes: Uint8Array }>, bytes: Uint8Array): Promise {
+ for (const [start, end] of edgeWindows(bytes.length)) {
+ let got: { status: number; bytes: Uint8Array }
+ try {
+ got = await read(`bytes=${start}-${end - 1}`)
+ } catch (e) {
+ throw new Error(`${label} is not readable by others from a browser (${errText(e)}): check public access and CORS`)
+ }
+ if (got.status !== 206 && got.status !== 200) throw new Error(`${label} answered HTTP ${got.status} to an anonymous read: the copy is not public`)
+ const slice = got.status === 200 ? got.bytes.subarray(start, end) : got.bytes
+ if (!bytesEqual(slice, bytes.subarray(start, end))) throw new Error(`${label} served bytes ${start}..${end} that differ from the artifact`)
+ }
+}
+
+/** Store and verify on one external profile; the URIs its manifest entry records. */
+async function storeExternal(p: StorageProfile, bytes: Uint8Array, hashHex: string): Promise {
+ const problem = profileProblem(p) ?? publishProblem(p)
+ if (problem) throw new Error(`the profile is not usable: ${problem}`)
+ const s = p.settings
+ if (s.kind === 's3') {
+ const key = artifactKey(s, hashHex)
+ const existing = await headObject(s, p.secrets, key).catch(() => null)
+ if (existing !== bytes.length) await putObject(s, p.secrets, key, bytes)
+ try {
+ await verifyS3(s, p, key, bytes, hashHex)
+ } catch (first) {
+ // A same-size but corrupt object at a content-addressed key would fail every push:
+ // re-upload once, unconditionally, and check again.
+ await putObject(s, p.secrets, key, bytes)
+ try {
+ await verifyS3(s, p, key, bytes, hashHex)
+ } catch (second) {
+ throw new Error(`${errText(second)} (also after a re-upload; first attempt: ${errText(first)})`)
+ }
+ }
+ await verifyPublic(`the public URL (${new URL(s.publicUrl).host})`, (range) => getPublic(s, key, range), bytes)
+ return [publicObjectUrl(s, key), s3Uri(s, key)]
+ }
+ if (s.kind === 'ipfs-kubo' || s.kind === 'ipfs-pinning-service') {
+ const cid = await addVerified(s, p.secrets, bytes)
+ if (s.kind === 'ipfs-pinning-service') await remotePin(s, p.secrets, cid, `dash-forge ${hashHex.slice(0, 16)}`)
+ if (s.gateway !== '') {
+ const r = await timedFetch(gatewayUrl(s.gateway, cid), { credentials: 'omit', cache: 'no-store' })
+ const got = await r.bytes()
+ if (!r.resp.ok || (await sha256Hex(got)) !== hashHex) throw new Error(`re-read of ipfs://${cid} through the gateway did not return the artifact`)
+ }
+ const uris = [`ipfs://${cid}`]
+ if (s.publicGateway !== '') {
+ const url = gatewayUrl(s.publicGateway, cid)
+ await verifyPublic(`the public gateway (${new URL(url).host})`, async (range) => {
+ const r = await timedFetch(url, { headers: { Range: range }, credentials: 'omit', cache: 'no-store' })
+ return { status: r.resp.status, bytes: await r.bytes() }
+ }, bytes)
+ uris.unshift(url)
+ }
+ return uris
+ }
+ throw new Error('not an external profile')
+}
+
+/**
+ * Every recorded URI, de-duplicated: the `platform://` locator first (released helpers read
+ * `uris[0]` of a storage-0 manifest as the chunk locator), then public https URLs, then the
+ * rest — each group in policy order (parity with forge-core `Replication::uris`).
+ */
+export function orderUris(groups: readonly (readonly string[])[]): string[] {
+ const buckets: string[][] = [[], [], []]
+ for (const uris of groups) {
+ for (const u of uris) {
+ const g = u.startsWith('platform://') ? 0 : /^https?:\/\//.test(u) ? 1 : 2
+ const bucket = buckets[g] as string[]
+ if (!bucket.includes(u)) bucket.push(u)
+ }
+ }
+ return buckets.flat()
+}
+
+/**
+ * Trim to the manifest's `uris` budget (8 × 300 bytes): private `s3://` locators go first
+ * (readers without that profile cannot use them); then an error, never a silent truncation.
+ * A URI no reader may fetch (plain http, this machine, a private network) is refused outright:
+ * it would be published on chain for every reader's browser to request.
+ */
+export function fitManifestUris(uris: readonly string[]): string[] {
+ const bad = uris.find((u) => !isRecordableUri(u))
+ if (bad !== undefined) throw new Error(`refusing to record ${bad} on chain: only public https addresses can be read by others`)
+ const fits = (list: readonly string[]): boolean => manifestUrisProblem(list) === null
+ if (fits(uris)) return [...uris]
+ const noS3 = uris.filter((u) => !u.startsWith('s3://'))
+ if (fits(noS3)) return noS3
+ throw new Error("the confirmed copies' URIs do not fit a manifest (at most 8, each up to 300 bytes); use shorter public URLs or fewer targets")
+}
+
+/** What {@link storeArtifact} needs besides the bytes. */
+export interface StoreOptions {
+ readonly policy: StoragePolicy | null
+ readonly profiles: readonly StorageProfile[]
+ /** Asked once, with the price, before anything is written to Platform. False stops it. */
+ readonly confirmPlatform: (q: PlatformQuestion) => Promise
+ readonly onStep?: (e: UploadEvent) => void
+}
+
+/**
+ * Store `bytes` under `policy`, verifying every copy, and return what the manifest records.
+ * Before the first Platform chunk the user is asked, with the price; declining throws
+ * {@link PlatformDeclinedError} (nothing configured) or {@link ReplicationError} (the policy
+ * needed the Platform copy). Fewer than `replicas` confirmations throws {@link ReplicationError}.
+ */
+export async function storeArtifact(
+ sdk: EvoSDK,
+ auth: WriteAuth,
+ repo: V2RepoRef,
+ bytes: Uint8Array,
+ opts: StoreOptions,
+): Promise {
+ const hashHex = await sha256Hex(bytes)
+ const step = opts.onStep ?? (() => undefined)
+ const confirmed: { target: string; uris: string[]; platform: boolean }[] = []
+ const failures: TargetFailure[] = []
+ let chunkCount = 0
+ let platformWritten = false
+ let platformAgreed: boolean | null = null
+
+ const fail = (target: string, e: unknown): void => {
+ failures.push({ target, reason: errText(e) })
+ step({ target, phase: 'failed', reason: errText(e) })
+ }
+ /** Ask once per push; every Platform write goes through this first. */
+ const agreePlatform = async (reason: string): Promise => {
+ if (platformAgreed === null) {
+ platformAgreed = await opts.confirmPlatform({ bytes: bytes.length, estimateCredits: estimateChunkCredits(bytes.length), reason })
+ }
+ return platformAgreed
+ }
+
+ const platformCopy = async (target: string): Promise => {
+ step({ target, phase: 'start' })
+ try {
+ const r = await putPlatformChunks(sdk, auth, repo, bytes, hashHex, (done, total, written) => {
+ if (written > 0) platformWritten = true
+ step({ target, phase: 'progress', done, total })
+ })
+ chunkCount = r.chunkCount
+ confirmed.push({ target, uris: [r.locator], platform: true })
+ step({ target, phase: 'done', uris: [r.locator] })
+ } catch (e) {
+ fail(target, e)
+ throw e
+ }
+ }
+
+ const policy = opts.policy
+ if (policy === null || policy.targets.length === 0) {
+ if (!(await agreePlatform('No storage is configured for browser pushes to this repo.'))) throw new PlatformDeclinedError()
+ await platformCopy('platform')
+ } else {
+ const byName = new Map(opts.profiles.map((p) => [p.name, p]))
+ const targets = policy.targets.map((name) => ({ name, profile: byName.get(name) }))
+ const external = targets.filter((t) => t.profile?.settings.kind !== 'platform')
+ const onChain = targets.filter((t) => t.profile?.settings.kind === 'platform')
+ await Promise.all(
+ external.map(async ({ name, profile }) => {
+ step({ target: name, phase: 'start' })
+ try {
+ if (!profile) throw new Error('no such storage profile in this browser')
+ const uris = await storeExternal(profile, bytes, hashHex)
+ confirmed.push({ target: name, uris, platform: false })
+ step({ target: name, phase: 'done', uris })
+ } catch (e) {
+ fail(name, e)
+ }
+ }),
+ )
+ const notMet = (): ReplicationError => new ReplicationError(policy.replicas, confirmed.map((c) => c.target), failures, platformWritten)
+ // Platform targets only if they can still make up the policy: never pay for chunks whose
+ // push is about to fail anyway — and never without the user's go-ahead on the price.
+ for (const [i, { name }] of onChain.entries()) {
+ if (confirmed.length + (onChain.length - i) < policy.replicas) break
+ if (!(await agreePlatform(`Your storage policy for this repo includes Dash Platform (${name}).`))) {
+ fail(name, new Error('storing on Platform was declined'))
+ throw notMet()
+ }
+ await platformCopy(name).catch(() => undefined)
+ }
+ if (confirmed.length < policy.replicas) {
+ if (!policy.platformFallback || confirmed.some((c) => c.platform)) throw notMet()
+ if (!(await agreePlatform(`Your storage did not confirm (${failures.map((f) => f.target).join(', ')}).`))) throw notMet()
+ try {
+ await platformCopy('platform (fallback)')
+ } catch {
+ throw notMet()
+ }
+ }
+ }
+
+ // Keep policy order within each URI group.
+ const order = policy?.targets ?? []
+ confirmed.sort((a, b) => order.indexOf(a.target) - order.indexOf(b.target))
+ const hasPlatform = confirmed.some((c) => c.platform)
+ return {
+ packHash: hashHex,
+ sizeBytes: bytes.length,
+ storage: hasPlatform ? 0 : 1,
+ chunkCount: hasPlatform ? chunkCount : 0,
+ uris: fitManifestUris(orderUris(confirmed.map((c) => c.uris))),
+ confirmed: confirmed.map((c) => c.target),
+ failures,
+ }
+}
diff --git a/forge-web/lib/storage/util.ts b/forge-web/lib/storage/util.ts
new file mode 100644
index 000000000..b7fe4c01c
--- /dev/null
+++ b/forge-web/lib/storage/util.ts
@@ -0,0 +1,117 @@
+/** Small helpers shared by the live test (`./probe`) and the upload path (`./upload`). */
+
+/** Whether `a` and `b` hold the same bytes. */
+export function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
+ return a.length === b.length && a.every((x, i) => x === b[i])
+}
+
+/** The message of a thrown value, as a row detail or a failure reason. */
+export function errText(e: unknown): string {
+ return e instanceof Error ? e.message : String(e)
+}
+
+/** A response is expected to begin within this long (a dead or silent host). */
+export const CONNECT_MS = 15_000
+/** A response body may pause this long between chunks before it is abandoned. */
+export const IDLE_MS = 120_000
+/**
+ * The slowest upload rate assumed when a request carries a body: `fetch` reports no upload
+ * progress, so the wait for the response grows with the body instead of being one fixed cap.
+ * 16 KiB/s is well below any usable uplink; a stall still ends within `CONNECT_MS + IDLE_MS`
+ * plus the body at that rate.
+ */
+const MIN_UPLOAD_BYTES_PER_S = 16 * 1024
+
+/** The request timed out (no answer, or the body stalled). */
+export class TimeoutError extends Error {
+ constructor(message: string) {
+ super(message)
+ this.name = 'TimeoutError'
+ }
+}
+
+/** A response whose body is read with an idle deadline re-armed on every chunk. */
+export interface TimedResponse {
+ readonly resp: Response
+ bytes(): Promise
+ text(): Promise
+ /** Done without the body (a HEAD, a status-only answer): clear the deadline, drop the body. */
+ discard(): void
+}
+
+/**
+ * `fetch` with progress deadlines rather than one total cap (forge-core's clients use a 15 s
+ * connect and a 120 s read timeout): the response must begin within {@link CONNECT_MS} (plus
+ * the upload time of `uploadBytes` at a conservative rate), then the body may stream for as
+ * long as it keeps moving, pausing at most {@link IDLE_MS}.
+ */
+export async function timedFetch(
+ url: string | URL,
+ init: RequestInit,
+ opts: { readonly uploadBytes?: number; readonly signal?: AbortSignal; readonly idleMs?: number } = {},
+): Promise {
+ const controller = new AbortController()
+ const idleMs = opts.idleMs ?? IDLE_MS
+ let timedOut: string | null = null
+ let timer: ReturnType | undefined
+ const arm = (ms: number, why: string): void => {
+ clearTimeout(timer)
+ timer = setTimeout(() => {
+ timedOut = why
+ controller.abort()
+ }, ms)
+ }
+ const onCancel = (): void => controller.abort()
+ opts.signal?.addEventListener('abort', onCancel)
+ const uploadMs = Math.ceil(((opts.uploadBytes ?? 0) / MIN_UPLOAD_BYTES_PER_S) * 1000)
+ arm(CONNECT_MS + uploadMs, `no answer within ${Math.round((CONNECT_MS + uploadMs) / 1000)}s`)
+ const done = (): void => {
+ clearTimeout(timer)
+ opts.signal?.removeEventListener('abort', onCancel)
+ }
+ let resp: Response
+ try {
+ resp = await fetch(url, { ...init, signal: controller.signal })
+ } catch (e) {
+ done()
+ if (timedOut !== null) throw new TimeoutError(timedOut)
+ throw e
+ }
+ const bytes = async (): Promise => {
+ try {
+ if (resp.body === null) return new Uint8Array(await resp.arrayBuffer())
+ const reader = resp.body.getReader()
+ const parts: Uint8Array[] = []
+ let total = 0
+ for (;;) {
+ arm(idleMs, `the answer stalled for ${idleMs / 1000}s`)
+ const { done: end, value } = await reader.read()
+ if (end) break
+ parts.push(value)
+ total += value.length
+ }
+ const out = new Uint8Array(total)
+ let at = 0
+ for (const p of parts) {
+ out.set(p, at)
+ at += p.length
+ }
+ return out
+ } catch (e) {
+ if (timedOut !== null) throw new TimeoutError(timedOut)
+ throw e
+ } finally {
+ done()
+ }
+ }
+ const discard = (): void => {
+ done()
+ resp.body?.cancel().catch(() => undefined)
+ }
+ return { resp, bytes, text: async () => new TextDecoder().decode(await bytes()), discard }
+}
+
+/** Printable ASCII only: header values (`fetch` throws on anything else, quoting the value). */
+export function isHeaderSafe(value: string): boolean {
+ return /^[\x20-\x7e]*$/.test(value)
+}
diff --git a/forge-web/lib/view/browse-fallback.test.ts b/forge-web/lib/view/browse-fallback.test.ts
index a59f36fd7..3ec6cee95 100644
--- a/forge-web/lib/view/browse-fallback.test.ts
+++ b/forge-web/lib/view/browse-fallback.test.ts
@@ -163,7 +163,7 @@ describe('startFallback with external-storage packs', () => {
const external = manifestFor(ext.pack, 1, {
storage: 1,
chunkCount: 0,
- uris: ['http://127.0.0.1:9000/forge-byo/pack', 's3://forge-byo/pack'],
+ uris: ['https://mirror.example/forge-byo/pack', 's3://forge-byo/pack'],
createdAt: 2,
documentId: 'b',
uploader: 'owner',
@@ -172,16 +172,16 @@ describe('startFallback with external-storage packs', () => {
const calls = stubFetch({})
const ctx = await startFallback(mockSdk(new Map([[platform.packHash, plat.pack]])), repo, [platform, external])
- expect(calls).toEqual(['http://127.0.0.1:9000/forge-byo/pack']) // s3:// is not browser-fetchable
+ expect(calls).toEqual(['https://mirror.example/forge-byo/pack']) // s3:// is not browser-fetchable
expect(ctx.unavailable).toHaveLength(1)
expect(ctx.unavailable?.[0]?.packHash).toBe(external.packHash)
- expect(ctx.unavailable?.[0]?.hosts).toEqual(['127.0.0.1:9000'])
+ expect(ctx.unavailable?.[0]?.hosts).toEqual(['mirror.example'])
expect(Array.from((await ctx.reader.readObject(plat.oid)).bytes)).toEqual(
Array.from(new TextEncoder().encode('on-chain content\n')),
)
// A needed object from the skipped pack gets a per-view error naming pack and storage.
await expect(ctx.reader.readObject(ext.oid)).rejects.toThrow(
- new RegExp(`${external.packHash.slice(0, 12)}.*127\\.0\\.0\\.1:9000`),
+ new RegExp(`${external.packHash.slice(0, 12)}.*mirror\\.example`),
)
// The trust ledger reports the gap: content is partial, never verified.
const checks = contentChecks(repo.contractId)
@@ -234,7 +234,7 @@ describe('startFallback with external-storage packs', () => {
const ext = blobPack('gone\n')
const platform = manifestFor(plat.pack, 1, { createdAt: 1, documentId: 'a' })
const external = (documentId: string): PackManifest =>
- manifestFor(ext.pack, 1, { storage: 1, uris: ['http://127.0.0.1:9000/p'], createdAt: 2, documentId })
+ manifestFor(ext.pack, 1, { storage: 1, uris: ['https://mirror.example/p'], createdAt: 2, documentId })
stubFetch({})
const ctx = await startFallback(
mockSdk(new Map([[platform.packHash, plat.pack]])),
@@ -283,7 +283,7 @@ describe('startFallback with external-storage packs', () => {
it('fails with the reasons when no live pack at all could be fetched', async () => {
const ext = blobPack('x\n')
- const external = manifestFor(ext.pack, 1, { storage: 1, uris: ['http://127.0.0.1:9000/p'] })
+ const external = manifestFor(ext.pack, 1, { storage: 1, uris: ['https://mirror.example/p'] })
stubFetch({})
await expect(
startFallback(mockSdk(new Map()), { kind: 'v1', contractId: 'fallback-none', ownerId: 'owner', name: '' }, [external]),
@@ -338,13 +338,13 @@ describe('startFallback with external-storage packs', () => {
const platform = manifestFor(plat.pack, 1, { createdAt: 0, documentId: 'a' })
const dead = (i: number): PackManifest => {
const p = blobPack(`gone ${i}\n`)
- return manifestFor(p.pack, 1, { storage: 1, uris: ['http://127.0.0.1:9000/p'], createdAt: i + 1, documentId: `x${i}` })
+ return manifestFor(p.pack, 1, { storage: 1, uris: ['https://mirror.example/p'], createdAt: i + 1, documentId: `x${i}` })
}
const calls = stubFetch({})
const sdk = mockSdk(new Map([[platform.packHash, plat.pack]]))
await startFallback(sdk, { kind: 'v1', contractId: 'fallback-dead', ownerId: 'o', name: '' }, [platform, dead(1)])
await startFallback(sdk, { kind: 'v1', contractId: 'fallback-dead-2', ownerId: 'o', name: '' }, [platform, dead(2)])
- expect(calls).toEqual(['http://127.0.0.1:9000/p'])
+ expect(calls).toEqual(['https://mirror.example/p'])
})
it('cancels external downloads when an on-chain pack fails', async () => {
@@ -378,12 +378,12 @@ describe('startFallback with external-storage packs', () => {
const packA = packFrame(concat(objHeader(T_BLOB, base.length), zlibSync(base)))
const delta = copyInsertDelta(base.length, target.length, 40, new TextEncoder().encode('cat\n'))
const packB = packFrame(concat(objHeader(T_REF_DELTA, delta.length), hexToBytes(baseOid), zlibSync(delta)))
- const a = manifestFor(packA, 1, { storage: 1, uris: ['http://127.0.0.1:9000/a'], createdAt: 0, documentId: 'a' })
+ const a = manifestFor(packA, 1, { storage: 1, uris: ['https://mirror.example/a'], createdAt: 0, documentId: 'a' })
const b = manifestFor(packB, 1, { createdAt: 1, documentId: 'b' })
stubFetch({})
await expect(
startFallback(mockSdk(new Map([[b.packHash, packB]])), { kind: 'v1', contractId: 'fallback-thin', ownerId: 'o', name: '' }, [a, b]),
- ).rejects.toThrow(new RegExp(`${a.packHash.slice(0, 12)}.*127\\.0\\.0\\.1:9000`))
+ ).rejects.toThrow(new RegExp(`${a.packHash.slice(0, 12)}.*mirror\\.example`))
})
it('expires a partial clone so a later view retries the mirrors', async () => {
diff --git a/forge-web/lib/view/browse-source.ts b/forge-web/lib/view/browse-source.ts
index 4aee1e3a8..0c877f961 100644
--- a/forge-web/lib/view/browse-source.ts
+++ b/forge-web/lib/view/browse-source.ts
@@ -42,6 +42,7 @@ import {
type V2RepoRef,
} from '../repo'
import { base64ToBytes, queryDocumentsWithProof } from '../sdk'
+import { isPublicHttpsUrl } from '../net'
import { externalSourceName, noteContentCheck, objectObserver } from './content-checks'
import { describePack, readGateways } from './storage-status'
@@ -286,8 +287,10 @@ export function externalFetchUrls(
): string[] {
const out: string[] = []
for (const uri of uris) {
- if (/^https?:\/\//i.test(uri)) out.push(uri)
- const ipfs = /^ipfs:\/\/(.+)$/i.exec(uri)
+ // A manifest is written by whoever pushed: a URL at this machine, a private network or
+ // over plain http would have every reader's browser request its own local services.
+ if (isPublicHttpsUrl(uri)) out.push(uri)
+ const ipfs = /^ipfs:\/\/([A-Za-z0-9]+)$/.exec(uri)
if (ipfs !== null) {
for (const gw of gateways) out.push(`${gw.replace(/\/+$/, '')}/ipfs/${ipfs[1]}`)
}
diff --git a/forge-web/tailwind.config.js b/forge-web/tailwind.config.js
index 60b0ebc89..491713f20 100644
--- a/forge-web/tailwind.config.js
+++ b/forge-web/tailwind.config.js
@@ -51,7 +51,11 @@ module.exports = {
DEFAULT: '#d97706', // degraded availability — icons, borders, tints; text on dark
700: '#b45309', // TEXT on light surfaces (5.02:1 on white; the base value is 3.19:1)
},
- danger: '#dc2626', // force-push, delete, failed verification
+ danger: {
+ DEFAULT: '#dc2626', // force-push, delete, failed verification: icons, borders, tints, text on light
+ 400: '#f87171', // TEXT on dark surfaces (the base value is 4.01:1 on anvil-950)
+ 700: '#b91c1c', // TEXT on light surfaces (the base value is 4.4:1 on anvil-100)
+ },
// Dash brand blue — identity/credits/network UI only. The brand value is for fills,
// tints and icons; it is under 4.5:1 as TEXT on every surface in both themes, so
// text uses the WCAG AA shades: `text-dash-600 dark:text-dash-400` (the same