From e6b43846535ef6a91f9481f800ec2e473996229f Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:02:37 +0800 Subject: [PATCH 001/262] ci(release): isolate Gitee mirror recovery --- .github/workflows/gitee-mirror.yml | 83 ++++++++++++++++++++++++ .github/workflows/repository-hygiene.yml | 10 --- docs/operations/github.md | 35 +++++++--- docs/operations/website.md | 3 +- docs/standards/versioning.md | 4 +- docs/todo.md | 14 ++-- scripts/mirror-release.mjs | 12 ++++ scripts/release-mirror-source.mjs | 50 ++++++++++++++ scripts/release-version.mjs | 3 +- tests/publish-release.test.ts | 27 +++++++- tests/release-automation.test.ts | 3 +- tests/release-mirror-source.test.ts | 70 ++++++++++++++++++++ 12 files changed, 283 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/gitee-mirror.yml create mode 100644 scripts/release-mirror-source.mjs create mode 100644 tests/release-mirror-source.test.ts diff --git a/.github/workflows/gitee-mirror.yml b/.github/workflows/gitee-mirror.yml new file mode 100644 index 000000000..92deb6419 --- /dev/null +++ b/.github/workflows/gitee-mirror.yml @@ -0,0 +1,83 @@ +name: Gitee mirror + +on: + workflow_run: + workflows: [CI] + types: [completed] + branches: [main] + workflow_dispatch: + inputs: + run_id: + description: Original CI run that published the current GitHub release + required: true + type: string + +permissions: + contents: read + actions: read + +concurrency: + group: gitee-mirror + cancel-in-progress: false + queue: max + +jobs: + mirror: + if: >- + github.repository == 'TNTcraftHIM/Piik' && github.ref == 'refs/heads/main' && + (github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'main' && + github.event.workflow_run.conclusion == 'success')) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + # Execute trusted main tooling; upstream artifacts are data, never code. + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-node@v6 + with: + node-version-file: .node-version + cache: npm + - run: npm ci --ignore-scripts + - name: Resolve the published source + id: source + run: node scripts/release-mirror-source.mjs "$PIIK_SOURCE_RUN_ID" + env: + GH_TOKEN: ${{ github.token }} + PIIK_SOURCE_RUN_ID: ${{ inputs.run_id || github.event.workflow_run.id }} + - name: Download original Server artifacts + if: steps.source.outputs.mirror == 'true' + uses: actions/download-artifact@v8 + with: + github-token: ${{ github.token }} + run-id: ${{ steps.source.outputs.run_id }} + pattern: piik-server*-${{ steps.source.outputs.revision }} + path: ${{ runner.temp }}/mirror-release + merge-multiple: true + - name: Download original App artifacts + if: steps.source.outputs.mirror == 'true' + uses: actions/download-artifact@v8 + with: + github-token: ${{ github.token }} + run-id: ${{ steps.source.outputs.run_id }} + pattern: piik-app-*-${{ steps.source.outputs.revision }} + path: ${{ runner.temp }}/mirror-release + merge-multiple: true + - name: Mirror and verify original packages + if: steps.source.outputs.mirror == 'true' + run: >- + node scripts/mirror-release.mjs + "$RUNNER_TEMP/mirror-release" "$PIIK_RELEASE_VERSION" "$PIIK_RELEASE_REVISION" + env: + GH_TOKEN: ${{ github.token }} + GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }} + PIIK_RELEASE_VERSION: ${{ steps.source.outputs.version }} + PIIK_RELEASE_REVISION: ${{ steps.source.outputs.revision }} + - name: Explain recovery + if: failure() && steps.source.outputs.mirror == 'true' + run: | + printf 'Mirror failed; GitHub publication and Website are independent. Retry this workflow with original CI run %s. If artifacts expired, recover the retained original build output; do not rebuild this version. See docs/operations/github.md.\n' "$PIIK_SOURCE_RUN_ID" >> "$GITHUB_STEP_SUMMARY" + env: + PIIK_SOURCE_RUN_ID: ${{ steps.source.outputs.run_id }} diff --git a/.github/workflows/repository-hygiene.yml b/.github/workflows/repository-hygiene.yml index 93f2d04ed..708e5c47c 100644 --- a/.github/workflows/repository-hygiene.yml +++ b/.github/workflows/repository-hygiene.yml @@ -290,7 +290,6 @@ jobs: # Local descriptors/manifests/checksums are validated before publishing. # Release assets contain packages only; checksums appear in the release body. - name: Publish packages with release notes and checksums - id: github_release run: >- node scripts/publish-release.mjs "$RUNNER_TEMP/publish-release" "$PIIK_RELEASE_VERSION" "$GITHUB_SHA" @@ -304,12 +303,3 @@ jobs: env: GH_TOKEN: ${{ github.token }} PIIK_RELEASE_VERSION: ${{ needs.validate.outputs.version }} - - name: Mirror verified release to Gitee - if: ${{ !cancelled() && steps.github_release.outcome == 'success' }} - run: >- - node scripts/mirror-release.mjs - "$RUNNER_TEMP/publish-release" "$PIIK_RELEASE_VERSION" "$GITHUB_SHA" - env: - GH_TOKEN: ${{ github.token }} - GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }} - PIIK_RELEASE_VERSION: ${{ needs.validate.outputs.version }} diff --git a/docs/operations/github.md b/docs/operations/github.md index 6adfd763d..bd28b8682 100644 --- a/docs/operations/github.md +++ b/docs/operations/github.md @@ -80,19 +80,34 @@ Image checks, version identity and retries follow ## Gitee Download Mirror [TNTcraftHIM/Piik](https://gitee.com/TNTcraftHIM/Piik) is a public README/Release -mirror. Source, documentation and issues stay on GitHub. The publishing job uses -the repository Actions secret `GITEE_TOKEN`, which is configured; rotate it there +mirror. Source, documentation and issues stay on GitHub. The **Gitee mirror** +workflow uses the repository Actions secret `GITEE_TOKEN`, which is configured; rotate it there when needed. Never put it in source, a remote URL or a distributed package. GitHub publication continues to use the workflow's scoped token. -After GitHub publication, the same job runs `scripts/mirror-release.mjs` on the -original artifacts. Upload failure leaves the Gitee release marked as a preview. -Rerun the failed publishing job to verify existing files and upload only missing -ones. GitHub's published release remains unchanged. Both publishers reject -changing a published package; use a corrected new version if its bytes are wrong. +After a successful main-push CI run, **Gitee mirror** downloads that run's original +artifacts and runs `scripts/mirror-release.mjs`. It skips runs that did not publish +the current GitHub release. GitHub publication and **Website** do not depend on +mirror success. A mirror failure stays visible in its own failed workflow; an +unfinished Gitee release remains a preview. -For a local retry, use the release revision's scripts and original complete build -output retained outside the checkout. Earlier versions can have a different +Rerun **Gitee mirror** to verify existing files and upload only missing ones. +For a separate retry, dispatch it on `main` with the **original CI run ID**: + +```sh +gh workflow run gitee-mirror.yml --repo TNTcraftHIM/Piik --ref main -f run_id=RUN_ID +``` + +The source must be a completed main-push run of this repository's CI workflow, +matching the latest published GitHub release. Manual recovery also accepts a +legacy CI run that failed after GitHub publication. The workflow executes trusted +main tooling and treats downloaded artifacts only as data. Old retries cannot +complete a mirror for a superseded release. Both publishers reject changing a +published package; use a corrected new version if its bytes are wrong. + +For a local retry, first let any active mirror workflow finish. Use the release +revision's scripts and original complete build output retained outside the +checkout. Earlier versions can have a different package set. While the publishing run's artifacts are available, download all packages into one directory (replace `RUN_ID`, `FULL_SOURCE_SHA` and `VERSION`): @@ -130,4 +145,6 @@ history. The upload script does not itself delete older attachments. - [Protected branch availability](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/about-protected-branches) - [Workflow concurrency and queued runs](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#example-queueing-multiple-pending-runs) +- [Workflow completion triggers and trust boundaries](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#workflow_run) +- [Downloading original artifacts from another run](https://github.com/actions/download-artifact#download-artifacts-from-other-workflow-runs-or-repositories) - [Immutable release workflow](https://docs.github.com/en/code-security/concepts/supply-chain-security/immutable-releases) diff --git a/docs/operations/website.md b/docs/operations/website.md index cbd8684b1..61fae6237 100644 --- a/docs/operations/website.md +++ b/docs/operations/website.md @@ -161,7 +161,8 @@ Verify the latest redirects and package checksums after releases. 4. Accepted main pushes run CI first; success triggers **Website**, which checks out that CI run's exact commit. Manual dispatch uses its selected `main` commit. Before publishing, both paths skip a build that is no longer current main; - dispatch again for the latest commit when needed. Download links resolve through each provider, so + dispatch again for the latest commit when needed. The separate Gitee mirror + workflow does not gate Website deployment. Download links resolve through each provider, so publishing a package or retrying its mirror needs no link refresh. Only `build/site/` is uploaded; website work does not package App/Server. Verify HTTPS, both languages, relative assets and per-platform download links. diff --git a/docs/standards/versioning.md b/docs/standards/versioning.md index d218dd5cc..ed20819d3 100644 --- a/docs/standards/versioning.md +++ b/docs/standards/versioning.md @@ -176,7 +176,9 @@ after all required artifacts and checks pass. The publisher verifies matching version, source SHA and checksums, then creates/uploads/publishes one GitHub draft. When there are no unpublished product changes, CI validates a development build and skips packaging and publication; it never assigns an old release version to -a new source SHA. The separate Website workflow updates the static site after CI. +a new source SHA. The separate Website and Gitee mirror workflows follow successful +main CI independently. Mirror failure must remain visible without blocking GitHub +publication or Website deployment; retries use the original publishing artifacts. Failed draft uploads can resume; published artifacts are never overwritten. An older draft retried after a newer release cannot take over `latest`. diff --git a/docs/todo.md b/docs/todo.md index ed06b4afa..a49c9558b 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -1,14 +1,16 @@ # Current TODO Ledger -Last reviewed: 2026-10-06 +Last reviewed: 2026-10-07 Only **Now** is executable. Product modules own behavior; Git/PRs own completed history. A parked idea is not implementation authority. ## Now -No active implementation work. Deferred proposals and evidence-dependent reports -remain below; release delivery is tracked by the pipeline and deployment records. +- [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance + are complete. Integrate through the next owner-approved publishing merge and + verify the separate workflow with its original CI artifacts. Mirror failure + must remain visible without blocking product CI or Website delivery. ## Held By Owner @@ -206,9 +208,9 @@ works, so a Rust alternative needs a concrete Go limitation. and retain Gitee; do not add a self-hosted mirror. Chrome still blocks the Gitee attachment when Referer is removed. Reopen for new evidence or a provider review; the warning remains unresolved. -10. **Release-operation policy.** Protected release environments, immutable - draft assets and changing mirror-failure policy remain unaccepted proposals; - evaluate their benefit before adding release machinery. +10. **Release-operation policy.** Protected release environments and immutable + draft assets remain unaccepted proposals; evaluate their benefit before + adding release machinery. 11. **Automatic local chat history.** Explicit TXT export covers manual retention. Reconsider automatic storage only with a stable room-incarnation identity, bounded retention and a clear delete control; reusable room codes must not diff --git a/scripts/mirror-release.mjs b/scripts/mirror-release.mjs index e881b73ae..bb569c950 100644 --- a/scripts/mirror-release.mjs +++ b/scripts/mirror-release.mjs @@ -15,6 +15,15 @@ const api = `https://gitee.com/api/v5/repos/${repository}`; const marker = ``; const token = process.env.GITEE_TOKEN?.trim(); +function requireLatestSource() { + const latest = JSON.parse(execFileSync("gh", ["api", `repos/${repository}/releases/latest`], + { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 30_000 })); + if (latest.tag_name !== version || latest.target_commitish !== revision || + latest.draft !== false || latest.prerelease !== false) { + throw new Error("Only the latest GitHub release may create or complete a Gitee mirror"); + } +} + async function fetchResponse(url, options, timeout) { const attempts = (options.method ?? "GET") === "GET" ? 3 : 1; for (let attempt = 1; ; attempt++) { @@ -88,6 +97,7 @@ if (option === "--dry-run") { throw new Error("Existing mirror release belongs to a different or unknown source revision"); } const body = `${source.body ?? ""}\n\nSource: https://github.com/${repository}/commit/${revision}\n\n${marker}\n`; + if (!release || release.prerelease) requireLatestSource(); if (!release) { const repo = await request(""); if (!repo || repo.private || !repo.default_branch) throw new Error("A public initialized Gitee mirror is required"); @@ -111,6 +121,8 @@ if (option === "--dry-run") { } // Gitee has no draft API. Its preview flag fences incomplete uploads from checks. if (release.prerelease) { + // Uploads may take minutes; a newer GitHub release must not be displaced. + requireLatestSource(); const completed = await request(`/releases/${release.id}`, "PATCH", { tag_name: version, name: source.name || `Piik ${version}`, body, prerelease: false, }); diff --git a/scripts/release-mirror-source.mjs b/scripts/release-mirror-source.mjs new file mode 100644 index 000000000..f3a8d0a14 --- /dev/null +++ b/scripts/release-mirror-source.mjs @@ -0,0 +1,50 @@ +import { execFileSync } from "node:child_process"; +import { appendFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +import { isReleaseVersion } from "./release-version.mjs"; + +const repository = "TNTcraftHIM/Piik"; +const gh = (path) => JSON.parse(execFileSync("gh", ["api", `repos/${repository}/${path}`], { + encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 30_000, +})); + +export function resolveMirrorSource(runId, automatic = false) { + if (!/^[1-9]\d*$/.test(runId) || !Number.isSafeInteger(Number(runId))) { + throw new Error("Provide the original CI run's numeric ID"); + } + const run = gh(`actions/runs/${runId}`); + if (run.id !== Number(runId) || run.repository?.full_name !== repository || + run.head_repository?.full_name !== repository || run.event !== "push" || + run.head_branch !== "main" || run.path !== ".github/workflows/repository-hygiene.yml" || + run.status !== "completed" || !/^[a-f0-9]{40}$/.test(run.head_sha) || + (automatic && run.conclusion !== "success")) { + throw new Error("Mirror artifacts must come from a completed main push in this repository's CI workflow"); + } + // A legacy CI run may be red because its mirror failed after GitHub published. + // The published release, not that aggregate result, owns manual retry identity. + const release = gh("releases/latest"); + if (release.draft !== false || release.prerelease !== false || !isReleaseVersion(release.tag_name) || + !/^[a-f0-9]{40}$/.test(release.target_commitish)) { + throw new Error("A published stable GitHub release with a full source SHA is required"); + } + if (release.target_commitish !== run.head_sha) { + if (automatic) return null; + throw new Error("This CI run does not match the latest GitHub release; use its original publishing run"); + } + return { run_id: String(run.id), version: release.tag_name, revision: run.head_sha }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const source = resolveMirrorSource(process.argv[2], process.env.GITHUB_EVENT_NAME === "workflow_run"); + const output = { mirror: source !== null, ...source }; + if (process.env.GITHUB_OUTPUT) { + appendFileSync(process.env.GITHUB_OUTPUT, + Object.entries(output).map(([key, value]) => `${key}=${value}\n`).join("")); + } + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, source + ? `Mirror ${source.version} from original [CI run ${source.run_id}](https://github.com/${repository}/actions/runs/${source.run_id}).\n` + : "This CI run did not publish the current GitHub release; no mirror work is needed.\n"); + } + console.log(JSON.stringify(output)); +} diff --git a/scripts/release-version.mjs b/scripts/release-version.mjs index cc1297695..a14bbf39b 100644 --- a/scripts/release-version.mjs +++ b/scripts/release-version.mjs @@ -18,7 +18,8 @@ const nonProductPaths = [ "site/**", "docs/**", "tests/**", ".agents/**", ".codex/**", ".githooks/**", "*.md", ".gitignore", ".env.example", "vitest.config.ts", "cmd/**/*.md", "internal/**/*.md", "native/**/*.md", "src/**/*.md", - ".github/ISSUE_TEMPLATE/**", ".github/pull_request_template.md", ".github/workflows/website.yml", + ".github/ISSUE_TEMPLATE/**", ".github/pull_request_template.md", + ".github/workflows/website.yml", ".github/workflows/gitee-mirror.yml", "scripts/build-website.mjs", "scripts/update-website-hero.mjs", "scripts/check-website-film.js", "scripts/check-docs.mjs", "scripts/check-container.mjs", "scripts/markdown-slug.mjs", "scripts/check-project-state.*", "scripts/install-hooks.*", "scripts/required-project-paths.txt", "scripts/tsconfig.json", diff --git a/tests/publish-release.test.ts b/tests/publish-release.test.ts index db528df62..5e5321c69 100644 --- a/tests/publish-release.test.ts +++ b/tests/publish-release.test.ts @@ -45,15 +45,26 @@ describe("Gitee mirror publication", () => { }); async function mirror(options: { existing?: "pending" | "published" | "unknown"; corrupt?: boolean; wrongSource?: boolean; wrongDigest?: boolean; missing?: boolean; + stale?: boolean; supersededDuringUpload?: boolean; fault?: { method: string; remaining: number; status: number | "timeout" } } = {}) { const mutations: string[] = []; const attachments = sourceAssets.map(({ name, size }) => ({ name, size, browser_download_url: download + name })); const present = options.existing === "pending" ? attachments.slice(0, 1) : options.missing ? [] : attachments; - command.mockReturnValue(JSON.stringify({ + const source = { ...release(false), html_url: `https://github.com/TNTcraftHIM/Piik/releases/tag/${version}`, target_commitish: options.wrongSource ? "b".repeat(40) : revision, assets: options.wrongDigest ? sourceAssets.map((asset) => ({ ...asset, digest: "sha256:wrong" })) : sourceAssets, - })); + }; + let latestReads = 0; + command.mockImplementation((_program, args) => { + if (args[1].endsWith("/releases/latest")) { + latestReads++; + if (options.stale || (options.supersededDuringUpload && latestReads > 1)) { + return JSON.stringify({ ...source, tag_name: "v1.1.0", target_commitish: "b".repeat(40) }); + } + } else expect(args[1]).toBe(`repos/TNTcraftHIM/Piik/releases/tags/${version}`); + return JSON.stringify(source); + }); vi.stubEnv("GITEE_TOKEN", "fixture-mirror-token"); vi.stubGlobal("fetch", vi.fn(async (input, init) => { const url = String(input); @@ -129,6 +140,18 @@ describe("Gitee mirror publication", () => { expect(await mirror({ existing: "published" })).toEqual({ mutations: [], failure: undefined }); }); + it("prevents stale creation or completion while allowing existing stable verification", async () => { + for (const existing of [undefined, "pending"] as const) { + const result = await mirror({ existing, stale: true }); + expect(result.failure).toContain("Only the latest GitHub release"); + expect(result.mutations).toEqual([]); + } + const superseded = await mirror({ existing: "pending", supersededDuringUpload: true }); + expect(superseded.failure).toContain("Only the latest GitHub release"); + expect(superseded.mutations).not.toContain("PATCH /releases/42"); + expect(await mirror({ existing: "published", stale: true })).toEqual({ mutations: [], failure: undefined }); + }); + it.each([503, "timeout"] as const)("retries transient read failures (%s) within a fixed bound", async (status) => { const fault = { method: "GET", remaining: 2, status }; expect((await mirror({ existing: "published", fault })).failure).toBeUndefined(); diff --git a/tests/release-automation.test.ts b/tests/release-automation.test.ts index adc7a70b9..08cbf1832 100644 --- a/tests/release-automation.test.ts +++ b/tests/release-automation.test.ts @@ -142,7 +142,8 @@ describe("release automation", () => { "scripts/build-website.mjs", "scripts/check-docs.mjs", "scripts/check-container.mjs", "scripts/markdown-slug.mjs", "scripts/release-notes.mjs", "scripts/publish-container.mjs", "scripts/app-gate-endpoint.ts", "scripts/client-gate-endpoint.ts", - "tests/room.test.ts", ".agents/skills/ponytail/SKILL.md", ".github/workflows/website.yml"]) { + "tests/room.test.ts", ".agents/skills/ponytail/SKILL.md", ".github/workflows/website.yml", + ".github/workflows/gitee-mirror.yml", "scripts/release-mirror-source.mjs"]) { mkdirSync(dirname(join(root, path)), { recursive: true }); writeFileSync(join(root, path), "Peripheral content"); } diff --git a/tests/release-mirror-source.test.ts b/tests/release-mirror-source.test.ts new file mode 100644 index 000000000..85b8c805e --- /dev/null +++ b/tests/release-mirror-source.test.ts @@ -0,0 +1,70 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { resolveMirrorSource } from "../scripts/release-mirror-source.mjs"; + +const { command } = vi.hoisted(() => ({ command: vi.fn() })); +vi.mock("node:child_process", () => ({ execFileSync: command })); + +const repository = "TNTcraftHIM/Piik"; +const revision = "a".repeat(40); +const run = { + id: 12345, repository: { full_name: repository }, head_repository: { full_name: repository }, + event: "push", head_branch: "main", head_sha: revision, + path: ".github/workflows/repository-hygiene.yml", status: "completed", conclusion: "success", +}; +const release = { tag_name: "v1.9.2", target_commitish: revision, draft: false, prerelease: false }; + +function api(source = run, latest = release) { + command.mockImplementation((program: string, args: string[]) => { + expect(program).toBe("gh"); + if (args[1] === `repos/${repository}/actions/runs/12345`) return JSON.stringify(source); + if (args[1] === `repos/${repository}/releases/latest`) return JSON.stringify(latest); + throw new Error(`Unexpected request: ${args.join(" ")}`); + }); +} + +beforeEach(() => { command.mockReset(); api(); }); + +describe("mirror source authority", () => { + it("binds automatic and manual retries to the published source run", () => { + const expected = { run_id: "12345", version: release.tag_name, revision }; + expect(resolveMirrorSource("12345", true)).toEqual(expected); + expect(resolveMirrorSource("12345")).toEqual(expected); + api({ ...run, conclusion: "failure" }); + expect(resolveMirrorSource("12345")).toEqual(expected); + expect(() => resolveMirrorSource("12345", true)).toThrow(/completed main push/); + }); + + it.each(["", "012345", "1e3", "-1", "12345\nmirror=true", "9007199254740992", undefined]) + ("rejects invalid run IDs before accessing GitHub: %s", (id) => { + expect(() => resolveMirrorSource(id)).toThrow(/numeric ID/); + expect(command).not.toHaveBeenCalled(); + }); + + it.each([ + { id: 99 }, { repository: { full_name: "fork/Piik" } }, + { head_repository: { full_name: "fork/Piik" } }, { event: "pull_request" }, + { event: "workflow_dispatch" }, { head_branch: "preview" }, + { path: ".github/workflows/other.yml" }, { status: "in_progress" }, { head_sha: "short" }, + ])("rejects artifacts from a different authority or unfinished run: %j", (patch) => { + api({ ...run, ...patch }); + expect(() => resolveMirrorSource("12345")).toThrow(/completed main push/); + expect(command).toHaveBeenCalledTimes(1); + }); + + it("skips automatic runs without a current release and refuses an explicit stale retry", () => { + api(run, { ...release, target_commitish: "b".repeat(40) }); + expect(resolveMirrorSource("12345", true)).toBeNull(); + expect(() => resolveMirrorSource("12345")).toThrow(/does not match the latest/); + }); + + it.each([{ draft: true }, { prerelease: true }, { tag_name: "preview" }, { target_commitish: "main" }]) + ("rejects an unverified release: %j", (patch) => { + api(run, { ...release, ...patch }); + expect(() => resolveMirrorSource("12345")).toThrow(/published stable GitHub release/); + }); + + it("does not turn an API failure into an empty successful mirror", () => { + command.mockImplementationOnce(() => { throw new Error("GitHub unavailable"); }); + expect(() => resolveMirrorSource("12345", true)).toThrow("GitHub unavailable"); + }); +}); From f213189e412c6590eb0febc34861748ff2dbbaf2 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:44:48 +0800 Subject: [PATCH 002/262] test(native): separate mailbox timing from codec work --- docs/todo.md | 9 ++++---- .../capture/windows/capture_control.test.cpp | 23 ++++++++----------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/docs/todo.md b/docs/todo.md index a49c9558b..18374419a 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -11,6 +11,11 @@ history. A parked idea is not implementation authority. are complete. Integrate through the next owner-approved publishing merge and verify the separate workflow with its original CI artifacts. Mirror failure must remain visible without blocking product CI or Website delivery. +- [ ] **Windows CI worker-check acceptance.** A constrained-CPU run reproduced + the worker test's timeout while its VP8 fixture was still encoding. The mailbox + lifecycle check now uses deterministic work; real encode/decode checks remain + separate. Verify the change on the hosted Windows runner before closing this + item. The earlier CI assertion lacks the phase needed to establish the same cause. ## Held By Owner @@ -65,10 +70,6 @@ These proposals remain deferred beyond the accepted interaction phase. These reports remain unmatched. A local repair or successful test is not proof of a reporter's cause; research owns the completed experiments and their limits. -- [ ] **Intermittent native capture self-test failure.** The worker check now - records its phase, original exception and per-output progress. A local pass - does not explain the earlier CI failure; use the next failed run's diagnostics - before changing deadlines or worker behavior. - [ ] **macOS and Linux physical capture.** Complete the remaining capture/audio checks in [verification status](./verification-status.md#device-evidence-boundary), including an Intel Mac with a usable hardware H.264 encoder, an ARM64 Linux diff --git a/native/capture/windows/capture_control.test.cpp b/native/capture/windows/capture_control.test.cpp index b94cb9946..9b825e9d8 100644 --- a/native/capture/windows/capture_control.test.cpp +++ b/native/capture/windows/capture_control.test.cpp @@ -54,12 +54,12 @@ void CheckOutputPipe() { CloseHandle(output); } -void CheckWorkers() { +void CheckMailboxWorkers() { struct Input { uint32_t index; }; using Mailbox = piik::capture::OutputMailbox; std::array mailboxes{Mailbox(90'000), Mailbox(300'000), Mailbox(1'200'000)}; std::array workers; - std::array last{}, count{}, retired{}; + std::array last{}, count{}, retired{}, bitrate{}; std::array recovery{}; std::mutex mutex; std::condition_variable changed; @@ -71,15 +71,10 @@ void CheckWorkers() { for (size_t layer = 0; layer < workers.size(); ++layer) { workers[layer] = std::thread([&, layer]() { try { - const uint32_t width = 160u << layer; - const uint32_t height = 90u << layer; - std::vector pixels(static_cast(width) * height * 3 / 2, 128); - std::unique_ptr encoder; for (;;) { const auto work = mailboxes[layer].Take(); if (work.action == Mailbox::Action::stop) break; if (work.action == Mailbox::Action::retire) { - encoder.reset(); std::lock_guard lock(mutex); ++retired[layer]; changed.notify_one(); @@ -89,14 +84,13 @@ void CheckWorkers() { entered.set_value(); released.wait(); } - if (!encoder) encoder = std::make_unique(width, height, 30, work.bitrate); - encoder->SetBitrate(work.bitrate); - const auto output = encoder->Encode(pixels.data(), width, - static_cast(work.input->index) * 10'000'000 / 30, work.recovery); + // Test mailbox ordering independently of codec speed. The separate + // encode/decode check below exercises real frames and keyframe output. if (!mailboxes[layer].Accept(work.generation)) continue; std::lock_guard lock(mutex); last[layer] = work.input->index; - recovery[layer] = output.key_frame; + bitrate[layer] = work.bitrate; + recovery[layer] = work.recovery; ++count[layer]; changed.notify_one(); } @@ -143,7 +137,7 @@ void CheckWorkers() { mailboxes[0].RequestKeyFrame(); release.set_value(); wait("pending-recovery", [&]() { return last[0] == 4; }); - assert(count[0] == 2 && recovery[0]); // New request survived the older keyframe; pending 2/3 were replaced. + assert(count[0] == 2 && recovery[0]); // New request survived older work; pending 2/3 were replaced. mailboxes[1].SetActive(false); mailboxes[2].SetActive(false); submit(5); @@ -154,6 +148,7 @@ void CheckWorkers() { submit(6); wait("output-reactivation", [&]() { return last[0] == 6 && last[1] == 6 && last[2] == 6; }); assert(recovery[0] && recovery[1] && recovery[2]); + assert(bitrate[0] == 30'000 && bitrate[1] == 300'000 && bitrate[2] == 1'200'000); const auto stopped = std::chrono::steady_clock::now(); for (auto& mailbox : mailboxes) mailbox.Stop(); for (auto& worker : workers) worker.join(); @@ -263,7 +258,7 @@ int main() { const auto decoded = decoder.Decode(output.bytes); assert(decoded.width == 160 && decoded.height == 90 && decoded.nv12.size() == pixels.size()); } - CheckWorkers(); + CheckMailboxWorkers(); CheckDroppedRecoveryInput(); CheckIndependentActivation(); CheckOutputPipe(); From bac1b672293038d610bce7c9ef4f42b0767ef6d5 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:53:50 +0800 Subject: [PATCH 003/262] docs: close verified Windows CI follow-up --- docs/todo.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/todo.md b/docs/todo.md index 18374419a..2f8dbeee4 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -11,11 +11,6 @@ history. A parked idea is not implementation authority. are complete. Integrate through the next owner-approved publishing merge and verify the separate workflow with its original CI artifacts. Mirror failure must remain visible without blocking product CI or Website delivery. -- [ ] **Windows CI worker-check acceptance.** A constrained-CPU run reproduced - the worker test's timeout while its VP8 fixture was still encoding. The mailbox - lifecycle check now uses deterministic work; real encode/decode checks remain - separate. Verify the change on the hosted Windows runner before closing this - item. The earlier CI assertion lacks the phase needed to establish the same cause. ## Held By Owner From c5507a077d2f18564ef6ccf9595d005bc8dbb405 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:04:28 +0800 Subject: [PATCH 004/262] feat(web): remember applied sharing quality preferences --- docs/guide/getting-started.md | 6 ++++ docs/guide/getting-started.zh-CN.md | 6 ++++ docs/standards/media-quality.md | 8 ++++++ docs/status.md | 8 ++++-- docs/todo.md | 42 +++++++++++++++++++--------- src/client/lib/quality-preference.ts | 28 +++++++++++++++++++ src/client/pages/HostPage.tsx | 10 ++++--- tests/host-quality-ownership.test.ts | 20 ++++++++++++- tests/quality-preference.test.ts | 36 ++++++++++++++++++++++++ 9 files changed, 143 insertions(+), 21 deletions(-) create mode 100644 src/client/lib/quality-preference.ts create mode 100644 tests/quality-preference.test.ts diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 7b03b958b..8d69e9c43 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -146,6 +146,12 @@ The site manages rooms and invitations; the App supplies native capture. If its operator has enabled SFU forwarding, turn off **Privacy mode** before sharing to allow that fallback. The project's online site uses P2P only. +## Picture and sound settings + +Open **Sharing settings** below the picture to adjust picture and audio quality. +Piik remembers your applied settings in this browser for this site. A new +temporary public-invite address has separate settings. + ## Add your voice While sharing, select **Microphone** below the picture and diff --git a/docs/guide/getting-started.zh-CN.md b/docs/guide/getting-started.zh-CN.md index 04d6bc1a9..9bfbb7595 100644 --- a/docs/guide/getting-started.zh-CN.md +++ b/docs/guide/getting-started.zh-CN.md @@ -129,6 +129,12 @@ Linux 原生采集还需要[系统组件](../../native/capture/linux/README.md) 如果管理员已启用 SFU 转发,房主可在开始分享前关闭 **隐私模式**,允许使用这条兜底线路。 项目提供的在线版仅使用 P2P。 +## 画面与声音设置 + +在画面下方打开 **分享设置**,调整画质和音频质量。 +Piik 会在当前浏览器中记住这个站点上已应用的设置; +公网邀请每次生成的新网址使用独立设置。 + ## 加上自己的声音 分享时,点击画面下方的 **麦克风**,并允许访问。 diff --git a/docs/standards/media-quality.md b/docs/standards/media-quality.md index 999f1e500..21b2033c4 100644 --- a/docs/standards/media-quality.md +++ b/docs/standards/media-quality.md @@ -101,6 +101,14 @@ contract. [Status](../status.md) indexes delivery and remaining acceptance. ## Video Profiles +The Host remembers explicitly applied picture and source-audio quality settings +in same-origin Browser storage. They seed a new page; authoritative room recovery +still owns an existing share. Failed or superseded changes and server snapshots +do not replace the preference. Storage failure falls back to session-only use. +This does not retain capture permission, device/session identity or microphone +activation. A different site or temporary public-link origin has separate +Browser preferences; this is not an App-wide configuration sync. + The three recommended profiles are ceilings, not delivery guarantees: | Profile | Target | Video bitrate ceiling | diff --git a/docs/status.md b/docs/status.md index 5070e0104..a2dab31e6 100644 --- a/docs/status.md +++ b/docs/status.md @@ -1,6 +1,6 @@ # Current Status -Last updated: 2026-10-06 +Last updated: 2026-10-07 This is the compact execution/deployment index. Product modules own behavior, [verification status](./verification-status.md) owns unresolved physical limits, @@ -59,8 +59,10 @@ SQLite schema 2 and the 1440p ceiling. Hosted empty-room retention and capacity guidance preserve indefinite retention by default. The accepted Demo policy is one hour without authenticated participants; the private deployment retains the default. Operator records own activation, database and postflight evidence. -4K and resolution/capability expansion are not planned for this phase. -Native Magicsock remains a separate experiment, outside product integration. +The owner reopened 2.0 candidate preparation, including reassessment of 4K, +codec expansion and Native Magicsock where demonstrated benefit justifies total +complexity. These experiments do not yet change the accepted contract or product +transport. Release remains subject to the owner's final acceptance. Immutable release artifacts own delivery completion; [TODO](./todo.md#now) owns remaining work. diff --git a/docs/todo.md b/docs/todo.md index 2f8dbeee4..56580e959 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -7,6 +7,35 @@ history. A parked idea is not implementation authority. ## Now +- [ ] **2.0 candidate review and acceptance.** The owner reopened investigation, + useful improvements and outstanding acceptance on 2026-10-07. Prepare one + coherent candidate and stop before publication. Preserve the existing product + model; evaluate held features against measurable benefit and total complexity + before accepting a changed contract. +- [ ] **App-wide sharing preferences (#460).** Same-origin Browser preferences + are implemented. Assess retaining explicit settings across changing App public + origins without eagerly claiming Native sessions, adding a second settings + owner or persisting live device/session authority. Keep the Browser scope clear + if an App-wide bridge is not justified. +- [ ] **Current issue evidence.** Review #452's public-link/H.264 report and + #453's cross-network failure against current startup and ICE behavior. Revisit + #432's motion-quality evidence and existing unmatched reports; do not equate a + successful local reproduction with resolving a different reporter's cause. +- [ ] **Current-contract runtime acceptance.** Exercise sharing, source/quality + replacement, reconnect, peer/SFU handoff and independent retirement. Repair + the obsolete optional `viewer-mbb` canary before using it as evidence. Review + responsive interaction and reader documentation against the accepted result; + keep unavailable physical-device checks explicit. +- [ ] **Deferred-work decisions.** Reassess transport, microphone pairing, + passive App attachment, hosting and remaining structural proposals in order + of demonstrated value. Record accepted scope in the owning modules before + implementation; do not turn every proposal into a release requirement. +- [ ] **4K, codec expansion and Native Magicsock assessment.** Reopened by the + owner on 2026-10-07 for 2.0 only where measured benefit justifies a simple + implementation. Review the [media extension assessment](./research/native-client-media.md#media-capability-extension-assessment) + and [transport comparison](./research/nat-traversal.md#integration-cost-and-recovery-controls) + against current main. No changed wire contract or product transport is accepted + merely by reopening the experiments. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance are complete. Integrate through the next owner-approved publishing merge and verify the separate workflow with its original CI artifacts. Mirror failure @@ -14,14 +43,6 @@ history. A parked idea is not implementation authority. ## Held By Owner -- [ ] **4K and resolution/capability expansion.** Not planned for this phase, - reaffirmed by the owner on 2026-10-06 because of cost relative to benefit. Keep the - public v23/v9/v7 contract and 1440p ceiling. Resume the extensibility audit and - [media extension assessment](./research/native-client-media.md#media-capability-extension-assessment) - only with a renewed benefit/cost decision. This includes unfinished - cross-platform capture and capability negotiation; local tests alone do not - authorize a protocol break or a 2.0 release. Retain recoverable Git evidence - for the experiment without requiring an active worktree. - [ ] **Browser-internal encoder work.** Paused by the owner on 2026-10-01. [Comparisons and Chromium traces](./research/browser-local-encoding-pool.md#sustained-h264-recovery) own the evidence for synchronous H264 initialization, native adaptation and @@ -40,11 +61,6 @@ history. A parked idea is not implementation authority. These proposals remain deferred beyond the accepted interaction phase. -- [ ] **Native Magicsock integration.** The [isolated comparison](./research/nat-traversal.md#integration-cost-and-recovery-controls) - establishes Go feasibility and relay-assisted recovery, but no additional - direct-reachability class in its tested matrix. Keep product transport unchanged. - Reopen integration only for a repeatable benefit on matched networks that - justifies the extra identity, readiness, packet-size and relay responsibilities. - [ ] **Managed Demo hosting.** Evaluate cost and public UDP support before any migration. The assessment remains outside the current product release; keep the existing P2P-only Demo deployment until a provider and operating boundary diff --git a/src/client/lib/quality-preference.ts b/src/client/lib/quality-preference.ts new file mode 100644 index 000000000..a5b25e8c8 --- /dev/null +++ b/src/client/lib/quality-preference.ts @@ -0,0 +1,28 @@ +import { + DEFAULT_QUALITY_SETTINGS, + qualitySettingsSchema, + type QualitySettings, +} from "../../shared/protocol"; + +const STORAGE_KEY = "piik:quality-preference:v1"; + +export function readPreferredQuality(): QualitySettings { + try { + const stored = window.localStorage.getItem(STORAGE_KEY); + const result = qualitySettingsSchema.safeParse(stored ? JSON.parse(stored) : null); + if (result.success) return result.data; + } catch { + // Storage may be unavailable; sharing still works with session settings. + } + return { ...DEFAULT_QUALITY_SETTINGS }; +} + +export function savePreferredQuality(settings: QualitySettings): void { + const result = qualitySettingsSchema.safeParse(settings); + if (!result.success) return; + try { + window.localStorage.setItem(STORAGE_KEY, JSON.stringify(result.data)); + } catch { + // Saving a preference must not turn an applied capture change into a failure. + } +} diff --git a/src/client/pages/HostPage.tsx b/src/client/pages/HostPage.tsx index 3cfda822f..4a3b0ae26 100644 --- a/src/client/pages/HostPage.tsx +++ b/src/client/pages/HostPage.tsx @@ -8,7 +8,6 @@ import { type FormEvent, } from "react"; import { - DEFAULT_QUALITY_SETTINGS, DEFAULT_ROUTE_POLICY, MAX_VIEWER_PASSWORD_LENGTH, viewerPasswordSchema, @@ -89,6 +88,7 @@ import { saveCreationProfile, type HostCreationProfile, } from "../lib/creation-profile"; +import { readPreferredQuality, savePreferredQuality } from "../lib/quality-preference"; import { createOpaqueId } from "../lib/opaque-id"; import { debugError, debugEvent, debugOperation } from "../lib/debug"; import { @@ -377,10 +377,10 @@ export function HostPage({ const copy = useCopy(); const { lang, vis, t, titleFrames } = copy; const [qualitySettings, setQualitySettings] = useState( - DEFAULT_QUALITY_SETTINGS, + readPreferredQuality, ); const [advancedQuality, setAdvancedQuality] = useState( - DEFAULT_QUALITY_SETTINGS, + qualitySettings, ); const [routePolicy, setRoutePolicy] = useState( () => ({ @@ -610,7 +610,7 @@ export function HostPage({ const sourceSwitchRef = useRef<{ replacingVideo?: MediaStreamTrack; audioOnly?: boolean } | null>(null); const qualityChangeRef = useRef(null); const pendingQualityChangeRef = useRef(null); - const qualitySettingsRef = useRef(DEFAULT_QUALITY_SETTINGS); + const qualitySettingsRef = useRef(qualitySettings); const routePolicyRef = useRef(routePolicy); const advancedQualityRef = useRef(advancedQuality); const videoCodecModeRef = useRef(videoCodecMode); @@ -1647,6 +1647,7 @@ export function HostPage({ return; } commitQuality(nextProfile); + savePreferredQuality(nextProfile); return; } if (qualityChangeRef.current) { @@ -1718,6 +1719,7 @@ export function HostPage({ } commitQuality(appliedProfile); + savePreferredQuality(appliedProfile); debugEvent("quality", "committed", { generation, applied: appliedProfile }); outcome = "applied"; if (nativeUpdate) { diff --git a/tests/host-quality-ownership.test.ts b/tests/host-quality-ownership.test.ts index 302ba4dc6..1beeadad1 100644 --- a/tests/host-quality-ownership.test.ts +++ b/tests/host-quality-ownership.test.ts @@ -101,7 +101,7 @@ function fixture(launchedByClient = true) { nativeEventCleanupRef: ref(null), nativeShareCleanupRef: ref(Promise.resolve()), peersRef: ref(new Map([["viewer", peer]])), hostProvisionalChildRef: ref(null), hostSfuRouteRef: ref(route), signalRef: ref({ setHostQualitySettings: vi.fn(), send: vi.fn() }), - setQualitySettings: vi.fn(), setAdvancedQuality: vi.fn(), setChangingQuality: vi.fn(), + setQualitySettings: vi.fn(), setAdvancedQuality: vi.fn(), setChangingQuality: vi.fn(), savePreferredQuality: vi.fn(), setNoticeValue: vi.fn(), setNoticeError: vi.fn(), readableError: hostActionErrorNotice, setDetails: vi.fn(), setNativeActive: vi.fn(), applyCaptureProfile: vi.fn(async (_stream: unknown, profile: QualitySettings) => { physical = profile; }), @@ -424,6 +424,21 @@ describe("Host room-link copy feedback", () => { }); describe("Host quality ownership", () => { + it("remembers an explicit pre-share choice without changing room or capture state", async () => { + const current = fixture(); + current.context.phase = "idle"; + await current.change(lower); + expect(current.savePreferredQuality).toHaveBeenCalledWith(lower); + expect(current.applyCaptureProfile).not.toHaveBeenCalled(); + expect(current.signalRef.current.setHostQualitySettings).not.toHaveBeenCalled(); + }); + + it("does not replace a local preference with reauthenticated room settings", () => { + const current = fixture(); + current.reauthenticate(); + expect(current.savePreferredQuality).not.toHaveBeenCalled(); + }); + it.each([true, false])("resumes the former raw audio input only after an accepted mixed-output resume: %s", (accepted) => { const current = fixture(false); const source = {} as MediaStream; @@ -1102,6 +1117,7 @@ describe("Host quality ownership", () => { expect(current.physical()).toEqual(lower); expect(current.qualitySettingsRef.current).toEqual(lower); expect(current.advancedQualityRef.current).toEqual(lower); + expect(current.savePreferredQuality).toHaveBeenCalledWith(lower); expect(current.peer.updateCaptureProfile).toHaveBeenCalledWith(lower); expect(optional.dispose).toHaveBeenCalledOnce(); expect(current.client.stopReceive).toHaveBeenCalledWith("share"); @@ -1116,6 +1132,7 @@ describe("Host quality ownership", () => { current.applyCaptureProfile.mockRejectedValue(new Error("capture rejected")); await current.change(lower); expect(current.physical()).toEqual(original); + expect(current.savePreferredQuality).not.toHaveBeenCalled(); expect(current.qualitySettingsRef.current).toEqual(original); expect(current.advancedQualityRef.current).toEqual(original); expect(optional.updateProfile).not.toHaveBeenCalled(); @@ -1160,6 +1177,7 @@ describe("Host quality ownership", () => { pending.resolve(); await changing; expect(current.qualitySettingsRef.current).toEqual(original); + expect(current.savePreferredQuality).not.toHaveBeenCalled(); expect(current.signalRef.current.setHostQualitySettings).not.toHaveBeenCalled(); }); diff --git a/tests/quality-preference.test.ts b/tests/quality-preference.test.ts new file mode 100644 index 000000000..282968565 --- /dev/null +++ b/tests/quality-preference.test.ts @@ -0,0 +1,36 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_QUALITY_SETTINGS } from "../src/shared/protocol"; +import { readPreferredQuality, savePreferredQuality } from "../src/client/lib/quality-preference"; + +afterEach(() => vi.unstubAllGlobals()); + +describe("sharing quality preference", () => { + it("retains explicit picture and audio settings across reads", () => { + const storage = new Map(); + vi.stubGlobal("window", { localStorage: { + getItem: (key: string) => storage.get(key) ?? null, + setItem: (key: string, value: string) => storage.set(key, value), + } }); + expect(readPreferredQuality()).toEqual(DEFAULT_QUALITY_SETTINGS); + const chosen = { ...DEFAULT_QUALITY_SETTINGS, resolution: "1440p" as const, + maxFramerate: 60, maxBitrate: 10_000_000, degradationPreference: "maintain-resolution" as const, + screenAudioQuality: "very-high" as const }; + savePreferredQuality(chosen); + expect(readPreferredQuality()).toEqual(chosen); + savePreferredQuality({ ...chosen, maxBitrate: -1 }); + expect(readPreferredQuality()).toEqual(chosen); + }); + + it.each(["{", "null", "{}", JSON.stringify({ ...DEFAULT_QUALITY_SETTINGS, resolution: "2160p" })])( + "uses valid defaults when saved settings are malformed or unsupported: %s", stored => { + vi.stubGlobal("window", { localStorage: { getItem: () => stored } }); + expect(readPreferredQuality()).toEqual(DEFAULT_QUALITY_SETTINGS); + }, + ); + + it("keeps sharing usable when Browser storage is denied", () => { + vi.stubGlobal("window", { get localStorage() { throw new Error("storage denied"); } }); + expect(readPreferredQuality()).toEqual(DEFAULT_QUALITY_SETTINGS); + expect(() => savePreferredQuality(DEFAULT_QUALITY_SETTINGS)).not.toThrow(); + }); +}); From a11393e05357e85cb136540ba13ee300ffcf695d Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:15:50 +0800 Subject: [PATCH 005/262] test(media): align topology gate with current sharing owners --- docs/todo.md | 8 +- scripts/peer-assisted-benchmark.ts | 225 +++++------------------------ 2 files changed, 39 insertions(+), 194 deletions(-) diff --git a/docs/todo.md b/docs/todo.md index 56580e959..0d40b53b4 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -22,8 +22,7 @@ history. A parked idea is not implementation authority. #432's motion-quality evidence and existing unmatched reports; do not equate a successful local reproduction with resolving a different reporter's cause. - [ ] **Current-contract runtime acceptance.** Exercise sharing, source/quality - replacement, reconnect, peer/SFU handoff and independent retirement. Repair - the obsolete optional `viewer-mbb` canary before using it as evidence. Review + replacement, reconnect, peer/SFU handoff and independent retirement. Review responsive interaction and reader documentation against the accepted result; keep unavailable physical-device checks explicit. - [ ] **Deferred-work decisions.** Reassess transport, microphone pairing, @@ -123,6 +122,9 @@ of a reporter's cause; research owns the completed experiments and their limits. the [runtime evidence](./research/cross-platform-client-runtime.md#public-invitation-startup). [#434](https://github.com/TNTcraftHIM/Piik/issues/434) establishes DNS refusal during edge discovery, not a cause shared by all startup reports. + [#452](https://github.com/TNTcraftHIM/Piik/issues/452) records both QUIC and TCP + connection timeouts; compare the same endpoint with/without its accelerator. + That startup log contains no evidence for its separate H.264 report. - [ ] **Viewer interruptions.** Obtain paired endpoint/upstream-relay diagnostics for established viewing returning to P2P connecting, including [#429](https://github.com/TNTcraftHIM/Piik/issues/429)'s SFU-to-P2P dropout. @@ -185,8 +187,6 @@ works, so a Rust alternative needs a concrete Go limitation. 1. **Representative device/network acceptance.** Resume the remaining matrix in [verification status](./verification-status.md#remaining-device-and-network-acceptance) when directed, preserving platform deferrals and serial physical checks. - Before reusing the optional `viewer-mbb` canary, replace its obsolete - Viewer-only trigger and capacity assertions with the sender-owned contract. 2. **Broader quality work.** Reopen from measured interruptions, failed recovery or healthy-sibling degradation at acceptable complexity. The [Browser pool comparison](./research/browser-local-encoding-pool.md#balanced-startup-and-recovery) diff --git a/scripts/peer-assisted-benchmark.ts b/scripts/peer-assisted-benchmark.ts index 9e41942c5..ecf076490 100644 --- a/scripts/peer-assisted-benchmark.ts +++ b/scripts/peer-assisted-benchmark.ts @@ -27,7 +27,6 @@ const PROFILE_SETTINGS = QUALITY_PROFILES; type ProfileId = QualityProfileId; type PageRole = "host" | "viewer"; type BenchmarkCodecMode = "vp8" | "auto" | "h264"; -type BenchmarkCanaryMode = "none" | "viewer-mbb"; const ROUTE_TIMING_KEYS = [ "queueWaitMs", @@ -54,14 +53,6 @@ type BenchmarkRouteTimingSummary = Record< RouteTimingDistribution >; -interface ViewerMbbCanaryResult { - kind: "viewer-mbb"; - status: "passed" | "failed"; - assertions: Record; - counters: Record; - failureCode?: "topology-unavailable" | "probe-unavailable"; -} - interface BenchmarkConfig { chromePath: string; viewerCounts: number[]; @@ -78,7 +69,6 @@ interface BenchmarkConfig { headless: boolean; noSandbox: boolean; qualityControlSmoke: boolean; - canaryMode: BenchmarkCanaryMode; } interface ExternalServerTarget { @@ -567,12 +557,6 @@ function parseBoolean(value: string | undefined, fallback: boolean, name: string throw new Error(`${name} must be true, false, 1, or 0`); } -function parseBenchmarkCanaryMode(value: string | undefined): BenchmarkCanaryMode { - const mode = value?.trim() || "none"; - if (mode === "none" || mode === "viewer-mbb") return mode; - throw new Error("BENCHMARK_CANARY must be none or viewer-mbb"); -} - function parseBenchmarkCodecMode( value: string | undefined, ): BenchmarkCodecMode { @@ -619,11 +603,8 @@ function parseBenchmarkConfig( "BENCHMARK_QUALITY_SMOKE requires a selected viewer count of at least 3", ); } - const canaryMode = parseBenchmarkCanaryMode(environment.BENCHMARK_CANARY); - if (canaryMode === "viewer-mbb" && !viewerCounts.includes(3)) { - throw new Error( - "BENCHMARK_CANARY=viewer-mbb requires a selected viewer count of 3", - ); + if (environment.BENCHMARK_CANARY && environment.BENCHMARK_CANARY !== "none") { + throw new Error("BENCHMARK_CANARY is obsolete; this gate covers topology and relay-loss recovery"); } const recoveryText = environment.BENCHMARK_RECOVERY_VIEWERS?.trim(); const recoveryViewerCount = recoveryText ? Number(recoveryText) : null; @@ -697,7 +678,6 @@ function parseBenchmarkConfig( "BENCHMARK_CHROME_NO_SANDBOX", ), qualityControlSmoke, - canaryMode, }; } @@ -1677,12 +1657,18 @@ function buildBenchmarkInitScript(options: { height: number; frameRate: number; expectedEndpointCap: number; + qualityControlSmoke?: boolean; }): string { const serialized = JSON.stringify(options); return `(() => { if (globalThis.__PIIK_BENCHMARK__) return; const options = ${serialized}; const expectedRoleCap = options.expectedEndpointCap; + // The parameter-readback arm exercises ordinary senders. Pool producers + // and their independent transport clocks have separate probe coverage. + if (options.qualityControlSmoke) { + Object.defineProperty(RTCRtpSender.prototype, "createEncodedStreams", { value: undefined, configurable: true }); + } if (options.clearHostRoom) { try { localStorage.removeItem("piik:host-room:v1"); } catch {} } @@ -1714,7 +1700,6 @@ function buildBenchmarkInitScript(options: { let transitionRevision = -1; let transitionPhase = null; let plannedRouteAssignment = null; - const canary = { prepareUpdates: 0, activeUpdates: 0, routeFailed: 0 }; const nativeSends = new WeakMap(); const routeTimingKeys = ${JSON.stringify(ROUTE_TIMING_KEYS)}; let routeDiagnosticRequested = false; @@ -1806,8 +1791,6 @@ function buildBenchmarkInitScript(options: { transitionRevision = revision; transitionPhase = phase; plannedRouteAssignment = assignment; - if (phase === "prepare") canary.prepareUpdates += 1; - else canary.activeUpdates += 1; if (phase === "active") { state.routeRevision = revision; state.routeAssignment = cloneRouteAssignment(assignment); @@ -1879,8 +1862,6 @@ function buildBenchmarkInitScript(options: { return; } else if (direction === "out" && message.type === "set-quality-settings") { state.qualitySettings = message.qualitySettings; - } else if (direction === "out" && message.type === "route-failed") { - canary.routeFailed += 1; } if (direction === "in" && message.type === "authenticated") { state.peerId = message.peerId; @@ -1923,8 +1904,6 @@ function buildBenchmarkInitScript(options: { const nativeSend = socket.send; nativeSends.set(socket, nativeSend); socket.send = function(data) { - let parsed = null; - if (typeof data === "string") { try { parsed = JSON.parse(data); } catch {} } const result = nativeSend.call(socket, data); handleSignalMessage(data, "out", socket); return result; @@ -1939,7 +1918,7 @@ function buildBenchmarkInitScript(options: { }, }); - function sendCanaryMessage(message) { + function sendDiagnosticMessage(message) { const socket = signalingSocket; const nativeSend = socket ? nativeSends.get(socket) : null; if (!socket || typeof nativeSend !== "function") return false; @@ -1958,7 +1937,7 @@ function buildBenchmarkInitScript(options: { return false; } routeDiagnosticRequested = true; - if (sendCanaryMessage({ type: "request-route-diagnostic" })) { + if (sendDiagnosticMessage({ type: "request-route-diagnostic" })) { return true; } routeDiagnosticRequested = false; @@ -2213,11 +2192,8 @@ function buildBenchmarkInitScript(options: { }; } - function canarySnapshot() { - return { ...canary, connectionCount: connections.length, maxActiveOutboundMediaEdges: state.maxActiveOutboundMediaEdges }; - } function senderDegradationPreferences() { - return connections.flatMap(({ connection }) => { + return networkConnections().flatMap(({ connection }) => { if ( connection.connectionState === "closed" || connection.connectionState === "failed" @@ -2227,15 +2203,12 @@ function buildBenchmarkInitScript(options: { .map((sender) => sender.getParameters().degradationPreference ?? null); }); } - function sendViewerQualityEvidence(message) { - return message?.type === "viewer-quality-evidence" && sendCanaryMessage(message); - } function markShareRequested() { state.shareRequestedAtEpochMs = Date.now(); } Object.defineProperty(globalThis, "__PIIK_BENCHMARK__", { configurable: false, - value: { sample, progress, snapshot, canarySnapshot, senderDegradationPreferences, sendViewerQualityEvidence, markShareRequested, requestRouteDiagnosticSnapshot, routeDiagnosticTimingSamples, stop: () => clearInterval(videoObserver) }, + value: { sample, progress, snapshot, senderDegradationPreferences, markShareRequested, requestRouteDiagnosticSnapshot, routeDiagnosticTimingSamples, stop: () => clearInterval(videoObserver) }, }); })();`; } @@ -2563,17 +2536,18 @@ async function startHost( "host controls", signal, ); + await evaluate(cdp, page, `(() => { + const settings = document.querySelector('button[aria-controls="host-advanced-door"]'); + if (!(settings instanceof HTMLButtonElement)) throw new Error('Sharing settings button missing'); + if (settings.getAttribute('aria-expanded') !== 'true') settings.click(); + })()`); + await waitForPage(cdp, page, "document.querySelector('#host-advanced-door .lr-tiles button')", + 5_000, "sharing settings", signal); if (codecMode !== "auto") { - await evaluate( - cdp, - page, - `(() => { - const advanced = document.querySelector('button[aria-controls="host-advanced-door"]'); - if (!(advanced instanceof HTMLButtonElement)) throw new Error('Advanced settings button missing'); - advanced.click(); - return true; - })()`, - ); + await evaluate(cdp, page, `(() => { + const technical = document.querySelector('#host-advanced-door details'); + if (technical instanceof HTMLDetailsElement && !technical.open) technical.querySelector('summary')?.click(); + })()`); await waitForPage( cdp, page, @@ -2612,6 +2586,9 @@ async function startHost( return true; })()`, ); + await waitForPage(cdp, page, "document.querySelector('.lr-source-option.is-browser')", + 5_000, "Browser capture choice", signal); + await evaluate(cdp, page, "document.querySelector('.lr-source-option.is-browser').click()"); await waitForPage( cdp, page, @@ -2896,7 +2873,10 @@ async function runQualityControlSmoke( 10_000, `${preference} sender parameter readback`, signal, - ); + ).catch(async error => { + const values = await evaluate(cdp, page, "globalThis.__PIIK_BENCHMARK__.senderDegradationPreferences()"); + throw new Error(`${errorMessage(error)}; expected ${expectedCount} sender(s), received ${JSON.stringify(values)}`); + }); return evaluate(cdp, page, `Boolean(${predicate})`); }), ); @@ -3207,123 +3187,6 @@ async function runRecovery( }; } -async function runViewerMbbCanary( - cdp: CdpConnection, - pages: PageHandle[], - signal: AbortSignal, -): Promise { - const failed = (failureCode: ViewerMbbCanaryResult["failureCode"]): ViewerMbbCanaryResult => ({ - kind: "viewer-mbb", status: "failed", assertions: {}, counters: {}, failureCode, - }); - let parentHandle: PageHandle | null = null; - try { - const observations = await Promise.all(pages.map((page) => progressSample(cdp, page))); - const byPeer = new Map(observations.flatMap((page) => page.peerId ? [[page.peerId, page] as const] : [])); - const viewers = observations.filter((page) => page.role === "viewer"); - const target = viewers.find((page) => { - const upstream = page.routeAssignment?.upstream; - const parent = upstream?.kind === "peer" ? byPeer.get(upstream.peerId) : null; - return page.routeRevision !== null && upstream?.kind === "peer" && - parent?.role === "viewer" && page.routeAssignment?.childPeerIds.length === 0; - }); - const candidate = viewers.find((page) => page !== target && page.peerId && - page.routeAssignment?.childPeerIds.length === 0 && hasAuthoritativeMediaUpstream(page) && - activeVideoEdgeCount(page, "send") === 0); - if (!target || !candidate || target.routeAssignment?.upstream.kind !== "peer") return failed("topology-unavailable"); - const parent = byPeer.get(target.routeAssignment.upstream.peerId); - if (!parent) return failed("topology-unavailable"); - const targetHandle = pages.find((page) => page.label === target.label); - parentHandle = pages.find((page) => page.label === parent.label) ?? null; - const candidateHandle = pages.find((page) => page.label === candidate.label); - const old = target.connections.find((connection) => - connection.hasInboundVideo && connection.connectionState === "connected" && - connection.remotePeerId === parent.peerId && connection.connectionId, - ); - if (!targetHandle || !parentHandle || !candidateHandle || !old || target.routeRevision === null || !target.peerId || !parent.peerId || !old.connectionId) return failed("probe-unavailable"); - const oldFrames = old.receiveTotals?.framesTotal ?? 0; - const revision = target.routeRevision; - const send = (page: PageHandle, method: string, message: unknown) => - evaluate(cdp, page, `globalThis.__PIIK_BENCHMARK__.${method}(${JSON.stringify(message)})`); - let windows = 0; - for (let sequence = 0; sequence < 3; sequence += 1) { - const windowSent = await send(targetHandle, "sendViewerQualityEvidence", { - type: "viewer-quality-evidence", guard: { connectionId: old.connectionId, routeRevision: revision }, - sequence, windowMs: 2_000, - metrics: { width: null, height: null, framesPerSecond: 0, bitrateKbps: 0, - packetsReceivedDelta: 100, packetsLostDelta: 0, jitterMs: 0, framesDecodedDelta: 0, - framesDroppedDelta: 0, decodeMsPerFrame: 0, freezeCountDelta: 0, freezeDurationMsDelta: null, - codec: null, codecProfile: null, codecParameters: null }, - }); - if (windowSent) windows += 1; - if (sequence < 2) await delay(2_100, signal); - } - let retained = false, provisional = false, promoted = false, sameIdentity = false, framesAdvanced = false; - let newId: string | null = null, newIndex: number | null = null; - let targetTelemetry = await canary(cdp, targetHandle); - let parentTelemetry = await canary(cdp, parentHandle); - let candidateTelemetry = await canary(cdp, candidateHandle); - const deadline = Date.now() + 15_000; - while (Date.now() < deadline) { - const current = await progressSample(cdp, targetHandle); - targetTelemetry = await canary(cdp, targetHandle); - parentTelemetry = await canary(cdp, parentHandle); - candidateTelemetry = await canary(cdp, candidateHandle); - const currentOld = current.connections.find((connection) => connection.connectionId === old.connectionId); - const currentNew = current.connections.find((connection) => connection.hasInboundVideo && connection.index !== old.index && connection.connectionState === "connected" && connection.connectionId !== old.connectionId); - if (currentOld?.connectionState === "connected" && currentNew) { - retained = true; provisional = true; newId = currentNew.connectionId; newIndex = currentNew.index; - } - framesAdvanced ||= (currentOld?.receiveTotals?.framesTotal ?? oldFrames) > oldFrames; - const upstream = current.routeAssignment?.upstream; - const upstreamPeerId = upstream?.kind === "peer" ? upstream.peerId : null; - promoted = current.routeRevision !== null && current.routeRevision !== revision && upstreamPeerId !== null && upstreamPeerId !== parent.peerId; - if (promoted) { - const active = current.connections.find((connection) => connection.hasInboundVideo && connection.connectionState === "connected" && connection.remotePeerId === upstreamPeerId); - sameIdentity = Boolean(active && ((newId && active.connectionId === newId) || (!newId && active.index === newIndex))); - } - if (promoted && sameIdentity && retained && framesAdvanced) break; - await delay(100, signal); - } - const assertions = { - syntheticViewerEvidence: windows === 3, - oldEdgeRetained: retained, - provisionalPcObserved: provisional, - promotedToViewerCandidate: promoted, - provisionalIdentityPromoted: sameIdentity, - oldMediaFramesAdvanced: framesAdvanced, - endpointCapRespected: Math.max(targetTelemetry.maxActiveOutboundMediaEdges, parentTelemetry.maxActiveOutboundMediaEdges, candidateTelemetry.maxActiveOutboundMediaEdges) <= 1, - noRouteFailed: targetTelemetry.routeFailed === 0 && parentTelemetry.routeFailed === 0 && candidateTelemetry.routeFailed === 0, - }; - return { kind: "viewer-mbb", status: Object.values(assertions).every(Boolean) ? "passed" : "failed", assertions, - counters: { pages: pages.length, evidenceWindows: windows, - routePrepareUpdates: targetTelemetry.prepareUpdates + parentTelemetry.prepareUpdates + candidateTelemetry.prepareUpdates, - routeActiveUpdates: targetTelemetry.activeUpdates + parentTelemetry.activeUpdates + candidateTelemetry.activeUpdates, - maxActiveOutboundEdges: Math.max(targetTelemetry.maxActiveOutboundMediaEdges, parentTelemetry.maxActiveOutboundMediaEdges, candidateTelemetry.maxActiveOutboundMediaEdges), - routeFailedMessages: targetTelemetry.routeFailed + parentTelemetry.routeFailed + candidateTelemetry.routeFailed }, - ...(Object.values(assertions).every(Boolean) ? {} : { failureCode: "probe-unavailable" as const }) }; - } catch { - return failed("probe-unavailable"); - } -} - -async function canary(cdp: CdpConnection, page: PageHandle): Promise<{ - prepareUpdates: number; activeUpdates: number; routeFailed: number; - maxActiveOutboundMediaEdges: number; -}> { - return evaluate(cdp, page, "globalThis.__PIIK_BENCHMARK__.canarySnapshot()"); -} - -function canaryChecks(result: ViewerMbbCanaryResult): RunCheck[] { - return [ - ...Object.entries(result.assertions).map(([name, passed]) => ({ - name: `viewer-mbb-${name}`, passed, actual: passed, expected: "true", - })), - ...Object.entries(result.counters).map(([name, actual]) => ({ - name: `viewer-mbb-counter-${name}`, passed: true, actual, expected: "sanitized counter", - })), - ]; -} - async function runCase( cdp: CdpConnection, baseUrl: string, @@ -3345,6 +3208,7 @@ async function runCase( height: captureResolution.height, frameRate: capture.maxFramerate, expectedEndpointCap: config.expectedEndpointCap, + qualityControlSmoke: config.qualityControlSmoke, }; const hostPage = await createPage(cdp, baseUrl, { ...commonInit, @@ -3413,21 +3277,6 @@ async function runCase( if (config.settleMs > 0) { await delay(config.settleMs, signal); } - if (config.canaryMode === "viewer-mbb" && viewerCount === 3) { - const canaryResult = await runViewerMbbCanary(cdp, pages, signal); - return { - viewerCount, - startedAt: new Date(startedAtMs).toISOString(), - completedAt: new Date().toISOString(), - status: canaryResult.status, - checks: canaryChecks(canaryResult), - summary: null, - routeTimingSummary, - routeTimingStatus, - samples: [], - recovery: { triggered: false }, - }; - } const measurementStartedAt = Date.now(); const deadline = measurementStartedAt + config.durationMs; do { @@ -3668,7 +3517,6 @@ async function main(): Promise { headless: config.headless, noSandbox: config.noSandbox, qualityControlSmoke: config.qualityControlSmoke, - canaryMode: config.canaryMode, chromeExecutable: basename(config.chromePath), output: config.outputPath ?? "stdout", }, @@ -3689,7 +3537,8 @@ async function main(): Promise { ? "The external server target is exercised as configured; the gate does not infer unavailable transports." : "The local runner does not start LiveKit; SFU consistency is reported only when an SFU route is actually observed.", "The harness emits raw gate fields and simple invariants; it does not implement a route score or runtime policy.", - "BENCHMARK_CANARY=viewer-mbb injects only sanitized control counters; it does not claim detector quality or network performance. Host-candidate and signaling-blackhole canaries remain deferred.", + "BENCHMARK_QUALITY_SMOKE disables encoded-stream support to verify ordinary-sender parameter propagation. Browser pool producer/clock settings are checked by browser-local-pool-probe.ts.", + "This gate measures initial topology and relay-loss recovery, not sender-quality convergence or synthetic Viewer-triggered handoffs.", ], runs: [], }; @@ -3724,10 +3573,7 @@ async function main(): Promise { PUBLIC_BASE_URL: baseUrl, ALLOWED_ORIGINS: baseUrl, ENDPOINT_MEDIA_COPY_CAPACITY: String(config.expectedEndpointCap), - MAX_VIEWERS_PER_ROOM: String(Math.max( - ...config.viewerCounts, - config.canaryMode === "viewer-mbb" ? 3 : 1, - )), + MAX_VIEWERS_PER_ROOM: String(Math.max(...config.viewerCounts)), // The loopback gate measures peer topology, never a relayed candidate. STUN_URLS: "", }, profileDirectory); @@ -3774,8 +3620,7 @@ async function main(): Promise { await authenticateExternalServer(cdp, externalServer, abortController.signal); } - const viewerCounts = config.canaryMode === "viewer-mbb" ? [3] : config.viewerCounts; - for (const viewerCount of viewerCounts) { + for (const viewerCount of config.viewerCounts) { abortController.signal.throwIfAborted(); console.error(`Running peer-topology loopback with ${viewerCount} viewer(s)`); report.runs.push( From 417daee0ddf9adbe259f0e59041eeb5bb9055e65 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:20:42 +0800 Subject: [PATCH 006/262] docs: clarify public invitation outbound connectivity --- docs/guide/troubleshooting.md | 5 +++++ docs/guide/troubleshooting.zh-CN.md | 4 ++++ docs/research/cross-platform-client-runtime.md | 8 ++++++++ 3 files changed, 17 insertions(+) diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 7203d1270..e13c3544c 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -217,6 +217,11 @@ from a page that opens but reports **No media route available**. firewall blocks Cloudflare Tunnel; adjust only relevant rules. Ask the administrator about a school or workplace network. +For a terminal timeout mentioning `:7844`, check outbound connections from +`cloudflared`, including proxy or game-accelerator rules. +[Cloudflare's connectivity checks](https://developers.cloudflare.com/cloudflare-one/networks/connectors/cloudflare-tunnel/troubleshoot-tunnels/connectivity-prechecks/) +cover UDP and TCP on this port. This does not require opening an inbound port on your router. + If it keeps failing, you can share through the [online site or an existing site](./getting-started.md#share-from-an-existing-site). Include the App version, failure time and [App diagnostic report](../../cmd/piik-app/README.md#diagnostics) in feedback. If the unavailable address is a self-hosted site rather than the App's diff --git a/docs/guide/troubleshooting.zh-CN.md b/docs/guide/troubleshooting.zh-CN.md index e69738f0f..d733b2e16 100644 --- a/docs/guide/troubleshooting.zh-CN.md +++ b/docs/guide/troubleshooting.zh-CN.md @@ -184,6 +184,10 @@ App 的 **公网邀请** 通过 Cloudflare Tunnel 提供临时网页地址。 3. **对照网络。** 可用手机热点重试,检查 DNS、代理或防火墙是否阻止 Cloudflare Tunnel。 只调整相关规则;学校或公司的网络应联系管理员。 +如果终端中的超时错误带有 `:7844`,请检查 `cloudflared` 的出站连接, +包括代理或游戏加速器的规则。[Cloudflare 连接检查](https://developers.cloudflare.com/cloudflare-one/networks/connectors/cloudflare-tunnel/troubleshoot-tunnels/connectivity-prechecks/)(英文) +说明了该端口的 UDP 和 TCP 要求,无需在路由器上开放入站端口。 + 反复失败时可先通过[在线版或已有站点](./getting-started.zh-CN.md#使用已有站点分享)分享。 反馈中请附上 App 版本、失败时间和 [App 诊断报告](../../cmd/piik-app/README.zh-CN.md#诊断)。 如果打不开的是自建站点而非 App 的临时地址,请联系该站点管理员检查服务。 diff --git a/docs/research/cross-platform-client-runtime.md b/docs/research/cross-platform-client-runtime.md index aee09a9f6..1af2c4ae1 100644 --- a/docs/research/cross-platform-client-runtime.md +++ b/docs/research/cross-platform-client-runtime.md @@ -190,6 +190,14 @@ own the dependency behavior. An early-exit fixture confirms fourth-start recover and bounded exhaustion; these checks do not identify a reporter's API or network failure or guarantee four starts when earlier attempts consume the deadline. +The public App log in [#452](https://github.com/TNTcraftHIM/Piik/issues/452), +reviewed on 2026-10-07, records a QUIC timeout followed by a TCP port-7844 +connect timeout within that deadline. Thus fallback ran; the report does not +establish a local port-80/443 collision. Its accelerator dependency remains a +network-path hypothesis requiring the same endpoint with/without the accelerator. +The attachment contains no capture start, so it cannot diagnose the separately +reported H.264 failure. + [HTTP 1033](https://developers.cloudflare.com/support/troubleshooting/http-status-codes/cloudflare-1xxx-errors/error-1033/) means Cloudflare cannot find a healthy tunnel connector. App readiness requires both the assigned origin and a registered connection; later reconnection belongs From 2c755bec242993627cc0db9b103dc8a1abc591a3 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:46:35 +0800 Subject: [PATCH 007/262] build(go): apply dependency security updates --- go.mod | 8 ++++---- go.sum | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index a933186be..eaef584e7 100644 --- a/go.mod +++ b/go.mod @@ -33,9 +33,9 @@ require ( github.com/pion/webrtc/v4 v4.2.19 github.com/thesyncim/gopus v0.1.1 go.uber.org/zap v1.28.0 - golang.org/x/crypto v0.54.0 - golang.org/x/mod v0.38.0 - golang.org/x/net v0.57.0 + golang.org/x/crypto v0.56.0 + golang.org/x/mod v0.40.0 + golang.org/x/net v0.58.0 golang.org/x/sync v0.22.0 golang.org/x/text v0.41.0 google.golang.org/protobuf v1.36.11 @@ -127,7 +127,7 @@ require ( golang.org/x/time v0.15.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect - google.golang.org/grpc v1.83.0 // indirect + google.golang.org/grpc v1.83.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/libc v1.75.6 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/go.sum b/go.sum index 0a1c4bcda..d5c5957be 100644 --- a/go.sum +++ b/go.sum @@ -339,14 +339,14 @@ go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= +golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 h1:ex206bKw+v3K0dm3andkrIF+ijyQKJG1pLgwQ2PYdQM= golang.org/x/exp v0.0.0-20260727155853-b88d891fe743/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -362,8 +362,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -406,8 +406,8 @@ golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -417,8 +417,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1: google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= -google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 2ac9aa5ba1b518f7109443ca7bf181b512cb5b94 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:46:39 +0800 Subject: [PATCH 008/262] test(media): allow serialized full-room admission --- scripts/peer-assisted-benchmark.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/peer-assisted-benchmark.ts b/scripts/peer-assisted-benchmark.ts index ecf076490..3348efb39 100644 --- a/scripts/peer-assisted-benchmark.ts +++ b/scripts/peer-assisted-benchmark.ts @@ -652,7 +652,8 @@ function parseBenchmarkConfig( connectionTimeoutMs: parseNumber( environment.BENCHMARK_CONNECTION_TIMEOUT_SECONDS, - 20, + // Includes the whole burst's queue: the room admits one child at a time. + 90, "BENCHMARK_CONNECTION_TIMEOUT_SECONDS", 3, 120, From b963ae474d7bf9ba172a2ace57cdd0c025ca6c35 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:10:45 +0800 Subject: [PATCH 009/262] feat(app): retain sharing preferences across invitation origins --- docs/guide/getting-started.md | 5 +- docs/guide/getting-started.zh-CN.md | 4 +- docs/standards/configuration.md | 4 + docs/standards/media-quality.md | 17 ++-- internal/app/app.go | 13 +++- internal/app/config/config.go | 6 +- internal/app/config/quality_preference.go | 76 ++++++++++++++++++ .../app/config/quality_preference_test.go | 74 ++++++++++++++++++ internal/app/loopback/quality_preference.go | 53 +++++++++++++ .../app/loopback/quality_preference_test.go | 78 +++++++++++++++++++ internal/app/loopback/server.go | 72 +++++++++-------- src/client/native/quality-preference.ts | 65 ++++++++++++++++ src/client/native/wire.ts | 1 + src/client/pages/HostPage.tsx | 47 ++++++++++- tests/host-quality-ownership.test.ts | 54 ++++++++++++- tests/native-quality-preference.test.ts | 68 ++++++++++++++++ 16 files changed, 586 insertions(+), 51 deletions(-) create mode 100644 internal/app/config/quality_preference.go create mode 100644 internal/app/config/quality_preference_test.go create mode 100644 internal/app/loopback/quality_preference.go create mode 100644 internal/app/loopback/quality_preference_test.go create mode 100644 src/client/native/quality-preference.ts create mode 100644 tests/native-quality-preference.test.ts diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 8d69e9c43..22c8f2665 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -149,8 +149,9 @@ sharing to allow that fallback. The project's online site uses P2P only. ## Picture and sound settings Open **Sharing settings** below the picture to adjust picture and audio quality. -Piik remembers your applied settings in this browser for this site. A new -temporary public-invite address has separate settings. +Piik remembers your applied settings for the next visit. Browser-only use remembers +them for the current site; opening Piik through an updated App also keeps them +when your public-invite address changes. ## Add your voice diff --git a/docs/guide/getting-started.zh-CN.md b/docs/guide/getting-started.zh-CN.md index 9bfbb7595..4f6ae532b 100644 --- a/docs/guide/getting-started.zh-CN.md +++ b/docs/guide/getting-started.zh-CN.md @@ -132,8 +132,8 @@ Linux 原生采集还需要[系统组件](../../native/capture/linux/README.md) ## 画面与声音设置 在画面下方打开 **分享设置**,调整画质和音频质量。 -Piik 会在当前浏览器中记住这个站点上已应用的设置; -公网邀请每次生成的新网址使用独立设置。 +Piik 会记住已应用的设置,供下次使用。仅用浏览器时,设置保存在当前站点; +通过新版 App 打开时,更换公网邀请网址也能保留设置。 ## 加上自己的声音 diff --git a/docs/standards/configuration.md b/docs/standards/configuration.md index 040ea29b6..d9b9c32e6 100644 --- a/docs/standards/configuration.md +++ b/docs/standards/configuration.md @@ -146,6 +146,10 @@ Command-line options select entry and local runtime behavior: Share quality, room access policy, language and theme are configured in the shared Web UI. Decorative motion follows the system's reduced-motion preference. +App-wide picture/audio quality preferences use `.quality`; removing +that optional file resets them on the next App launch. It contains no device or +room credentials. The [media owner](./media-quality.md#video-profiles) defines when +preferences apply; older Apps ignore the file and keep reading `client.json`. ## Diagnostics diff --git a/docs/standards/media-quality.md b/docs/standards/media-quality.md index 21b2033c4..d8d81387d 100644 --- a/docs/standards/media-quality.md +++ b/docs/standards/media-quality.md @@ -101,13 +101,16 @@ contract. [Status](../status.md) indexes delivery and remaining acceptance. ## Video Profiles -The Host remembers explicitly applied picture and source-audio quality settings -in same-origin Browser storage. They seed a new page; authoritative room recovery -still owns an existing share. Failed or superseded changes and server snapshots -do not replace the preference. Storage failure falls back to session-only use. -This does not retain capture permission, device/session identity or microphone -activation. A different site or temporary public-link origin has separate -Browser preferences; this is not an App-wide configuration sync. +The Host remembers explicitly applied picture and source-audio quality settings. +Browser-only use stores them per origin. An App-opened page uses the running, +authorized App's optional preference store when available, including across +temporary public-link origins; older or unavailable Apps retain Browser storage. +These preferences seed a new page, never an active share. Explicit edits and +authoritative room recovery win over a late preference read. Failed or superseded +changes and server snapshots do not replace the preference; pending saves retain +only the latest applied choice. Storage failure never fails capture or startup. +This does not retain capture permission, device/session identity, routing policy +or microphone activation, and reading settings claims no Native media session. The three recommended profiles are ceilings, not delivery guarantees: diff --git a/internal/app/app.go b/internal/app/app.go index f65036af5..c70859f3b 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -158,11 +158,16 @@ func Run(ctx context.Context, options Options) (returnedErr error) { "softwareVP8", nativeMedia.capabilities.SoftwareVP8, diagnostics.Error(nativeMedia.discoveryErr)) } recordContext(options, config) + qualityPreference, preferenceErr := appconfig.LoadQualityPreference(configPath) + if preferenceErr != nil { + slog.Debug("piik-client", "event", "quality-preference-load-failed", diagnostics.Error(preferenceErr)) + } control, err := loopback.Start(ctx, loopback.Options{ - AllowedOrigins: allowedOrigins(config.Site, options.Port), - NativeMedia: nativeMedia.capabilities, - NewControl: nativeMedia.controlFactory(), - Presentation: options.console.setLanguage, + AllowedOrigins: allowedOrigins(config.Site, options.Port), + NativeMedia: nativeMedia.capabilities, + NewControl: nativeMedia.controlFactory(), + Presentation: options.console.setLanguage, + QualityPreference: qualityPreference, }) if err != nil { return fmt.Errorf("Piik App native control could not start: %w", err) diff --git a/internal/app/config/config.go b/internal/app/config/config.go index 62583a33d..d6b096d26 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -72,7 +72,11 @@ func Save(path string, config Config) error { if err != nil { return err } - if err = os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return saveFile(path, payload) +} + +func saveFile(path string, payload []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return fmt.Errorf("create App configuration directory: %w", err) } // Keep the last readable configuration until the replacement is complete. diff --git a/internal/app/config/quality_preference.go b/internal/app/config/quality_preference.go new file mode 100644 index 000000000..e289a0ca1 --- /dev/null +++ b/internal/app/config/quality_preference.go @@ -0,0 +1,76 @@ +package config + +import ( + "encoding/json" + "errors" + "io" + "os" + "sync" + + "github.com/TNTcraftHIM/Piik/internal/server/protocol" +) + +const MaxQualityPreferenceBytes = 1024 + +// QualityPreference owns optional App-wide settings, not a live capture profile. +// A separate file leaves the strict configuration readable by published Apps. +type QualityPreference struct { + mu sync.Mutex + path string + value *protocol.QualitySettings +} + +// A missing or unreadable preference must not prevent the App from starting. +// Even on error the returned store can accept the user's next explicit choice. +func LoadQualityPreference(configPath string) (*QualityPreference, error) { + p := &QualityPreference{path: configPath + ".quality"} + file, err := os.Open(p.path) + if errors.Is(err, os.ErrNotExist) { + return p, nil + } + if err != nil { + return p, err + } + defer file.Close() + payload, err := io.ReadAll(io.LimitReader(file, MaxQualityPreferenceBytes+1)) + if err != nil { + return p, err + } + var quality protocol.QualitySettings + if len(payload) > MaxQualityPreferenceBytes || json.Unmarshal(payload, &quality) != nil { + return p, errors.New("invalid saved sharing quality") + } + p.value = &quality + return p, nil +} + +func (p *QualityPreference) Read() *protocol.QualitySettings { + p.mu.Lock() + defer p.mu.Unlock() + if p.value == nil { + return nil + } + value := *p.value + return &value +} + +func (p *QualityPreference) Save(quality protocol.QualitySettings) error { + payload, err := json.Marshal(quality) + if err != nil { + return err + } + // Reuse the wire validator instead of a second list of accepted settings. + if err := json.Unmarshal(payload, &quality); err != nil { + return err + } + p.mu.Lock() + defer p.mu.Unlock() + if p.value != nil && *p.value == quality { + return nil + } + if err := saveFile(p.path, append(payload, '\n')); err != nil { + return err + } + p.value = &quality + return nil +} diff --git a/internal/app/config/quality_preference_test.go b/internal/app/config/quality_preference_test.go new file mode 100644 index 000000000..10527de3b --- /dev/null +++ b/internal/app/config/quality_preference_test.go @@ -0,0 +1,74 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/TNTcraftHIM/Piik/internal/server/protocol" +) + +func TestQualityPreferencePersistsWithoutChangingAppConfiguration(t *testing.T) { + path := filepath.Join(t.TempDir(), "client.json") + if _, err := LoadOrCreate(path); err != nil { + t.Fatal(err) + } + original, _ := os.ReadFile(path) + p, err := LoadQualityPreference(path) + if err != nil || p.Read() != nil { + t.Fatalf("initial preference: %v, %v", p.Read(), err) + } + quality := protocol.DefaultQualitySettings + quality.Resolution, quality.ScreenAudioQuality = "1440p", "very-high" + if err := p.Save(quality); err != nil { + t.Fatal(err) + } + loaded, err := LoadQualityPreference(path) + if err != nil || loaded.Read() == nil || *loaded.Read() != quality { + t.Fatalf("restored: %v, %v", loaded.Read(), err) + } + copy := loaded.Read() + copy.Resolution = "480p" + if *loaded.Read() != quality { + t.Fatal("caller modified stored settings") + } + current, _ := os.ReadFile(path) + if string(current) != string(original) { + t.Fatal("preference changed strict App configuration") + } + invalid := quality + invalid.Resolution = "2160p" + if p.Save(invalid) == nil || *p.Read() != quality { + t.Fatal("invalid choice changed preference") + } + if err := os.Remove(path + ".quality"); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(path+".quality", 0700); err != nil { + t.Fatal(err) + } + quality.Resolution = "720p" + if p.Save(quality) == nil || p.Read().Resolution != "1440p" { + t.Fatal("failed write replaced saved preference") + } +} + +func TestUnavailableQualityPreferenceCanAcceptTheNextChoice(t *testing.T) { + path := filepath.Join(t.TempDir(), "client.json") + for _, payload := range []string{"null", "{}", "{", strings.Repeat(" ", MaxQualityPreferenceBytes+1)} { + if err := os.WriteFile(path+".quality", []byte(payload), 0600); err != nil { + t.Fatal(err) + } + p, err := LoadQualityPreference(path) + if err == nil || p == nil || p.Read() != nil { + t.Fatalf("invalid preference: %v, %v", p, err) + } + if err := p.Save(protocol.DefaultQualitySettings); err != nil { + t.Fatal(err) + } + if p.Read() == nil { + t.Fatal("choice was not saved") + } + } +} diff --git a/internal/app/loopback/quality_preference.go b/internal/app/loopback/quality_preference.go new file mode 100644 index 000000000..2f8527ff4 --- /dev/null +++ b/internal/app/loopback/quality_preference.go @@ -0,0 +1,53 @@ +package loopback + +import ( + "encoding/json" + "io" + "log/slog" + "net/http" + + appconfig "github.com/TNTcraftHIM/Piik/internal/app/config" + "github.com/TNTcraftHIM/Piik/internal/server/protocol" +) + +func (server *Server) handleQualityPreference(response http.ResponseWriter, request *http.Request) { + if server.qualityPreference == nil { + http.NotFound(response, request) + return + } + response.Header().Set("Cache-Control", "no-store") + if request.Method == http.MethodOptions { + response.Header().Set("Access-Control-Allow-Methods", "GET, PUT") + response.Header().Set("Access-Control-Allow-Headers", "Content-Type, X-Piik-Instance") + allowPrivateNetwork(response, request) + response.WriteHeader(http.StatusNoContent) + return + } + if request.Method != http.MethodGet && request.Method != http.MethodPut { + response.Header().Set("Allow", "GET, PUT, OPTIONS") + http.Error(response, "method not allowed", http.StatusMethodNotAllowed) + return + } + // Origin/Host are checked by ServeHTTP. Bind reads and writes to this App run. + if request.Header.Get("X-Piik-Instance") != server.endpoint.InstanceToken { + http.Error(response, "invalid App instance", http.StatusForbidden) + return + } + if request.Method == http.MethodGet { + writeJSON(response, http.StatusOK, server.qualityPreference.Read()) + return + } + request.Body = http.MaxBytesReader(response, request.Body, appconfig.MaxQualityPreferenceBytes) + decoder := json.NewDecoder(request.Body) + var quality protocol.QualitySettings + if decoder.Decode(&quality) != nil || decoder.Decode(&struct{}{}) != io.EOF { + http.Error(response, "invalid sharing quality", http.StatusBadRequest) + return + } + if err := server.qualityPreference.Save(quality); err != nil { + slog.Debug("piik-client", "event", "quality-preference-save-failed", "error", err) + http.Error(response, "sharing preference could not be saved", http.StatusInternalServerError) + return + } + response.WriteHeader(http.StatusNoContent) +} diff --git a/internal/app/loopback/quality_preference_test.go b/internal/app/loopback/quality_preference_test.go new file mode 100644 index 000000000..c5d167243 --- /dev/null +++ b/internal/app/loopback/quality_preference_test.go @@ -0,0 +1,78 @@ +package loopback + +import ( + "encoding/json" + "io" + "net/http" + "path/filepath" + "strings" + "testing" + + appconfig "github.com/TNTcraftHIM/Piik/internal/app/config" + "github.com/TNTcraftHIM/Piik/internal/server/protocol" +) + +func TestQualityPreferenceRequiresCurrentOriginAndInstanceWithoutClaimingControl(t *testing.T) { + p, err := appconfig.LoadQualityPreference(filepath.Join(t.TempDir(), "client.json")) + if err != nil { + t.Fatal(err) + } + server := startTestServerWithOptions(t, Options{AllowedOrigins: []string{testOrigin}, QualityPreference: p}) + endpoint := server.Endpoint() + response := doHealthRequest(t, endpoint, endpoint.Host, testOrigin) + var health Health + if err := json.NewDecoder(response.Body).Decode(&health); err != nil { + t.Fatal(err) + } + response.Body.Close() + if !health.QualityPreference { + t.Fatal("optional preference was not advertised") + } + payload, _ := json.Marshal(protocol.DefaultQualitySettings) + for _, input := range []struct { + method, origin, token, body string + status int + }{ + {"OPTIONS", testOrigin, "", "", http.StatusNoContent}, + {"OPTIONS", "https://other.example", "", "", http.StatusForbidden}, + {"GET", testOrigin, "", "", http.StatusForbidden}, + {"PUT", testOrigin, "old-instance", string(payload), http.StatusForbidden}, + {"PUT", "https://other.example", endpoint.InstanceToken, string(payload), http.StatusForbidden}, + {"PUT", testOrigin, endpoint.InstanceToken, `{}`, http.StatusBadRequest}, + {"PUT", testOrigin, endpoint.InstanceToken, `null`, http.StatusBadRequest}, + {"PUT", testOrigin, endpoint.InstanceToken, string(payload) + `{}`, http.StatusBadRequest}, + {"PUT", testOrigin, endpoint.InstanceToken, strings.Repeat(" ", 1025) + string(payload), http.StatusBadRequest}, + {"POST", testOrigin, endpoint.InstanceToken, string(payload), http.StatusMethodNotAllowed}, + {"PUT", testOrigin, endpoint.InstanceToken, string(payload), http.StatusNoContent}, + {"GET", testOrigin, endpoint.InstanceToken, "", http.StatusOK}, + } { + request, _ := http.NewRequest(input.method, endpoint.URL+"/quality-preference", strings.NewReader(input.body)) + request.Header.Set("Origin", input.origin) + request.Header.Set("X-Piik-Instance", input.token) + request.Header.Set("Access-Control-Request-Private-Network", "true") + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(response.Body) + response.Body.Close() + if response.StatusCode != input.status { + t.Fatalf("%s: status %d, want %d", input.method, response.StatusCode, input.status) + } + if input.status == http.StatusOK { + var quality protocol.QualitySettings + if err := json.Unmarshal(body, &quality); err != nil || quality != protocol.DefaultQualitySettings { + t.Fatalf("saved quality: %v, %v", quality, err) + } + } + if input.method == "OPTIONS" && input.status == http.StatusNoContent && + (response.Header.Get("Access-Control-Allow-Methods") != "GET, PUT" || response.Header.Get("Access-Control-Allow-Private-Network") != "true") { + t.Fatal("invalid preflight") + } + } + server.mu.Lock() + defer server.mu.Unlock() + if server.claimed != 0 { + t.Fatal("preference claimed media control") + } +} diff --git a/internal/app/loopback/server.go b/internal/app/loopback/server.go index a2f4e66e8..8efc580d0 100644 --- a/internal/app/loopback/server.go +++ b/internal/app/loopback/server.go @@ -16,6 +16,7 @@ import ( "sync" "time" + appconfig "github.com/TNTcraftHIM/Piik/internal/app/config" "github.com/coder/websocket" ) @@ -29,12 +30,13 @@ const ( ) type Options struct { - PortStart int - PortEnd int - AllowedOrigins []string - NativeMedia NativeMediaCapabilities - NewControl func() ControlSession - Presentation func(language string) + PortStart int + PortEnd int + AllowedOrigins []string + NativeMedia NativeMediaCapabilities + NewControl func() ControlSession + Presentation func(language string) + QualityPreference *appconfig.QualityPreference } type ControlSession interface { @@ -61,11 +63,12 @@ type Endpoint struct { } type Health struct { - Protocol int `json:"protocol"` - Service string `json:"service"` - Port int `json:"port"` - InstanceToken string `json:"instanceToken"` - NativeMedia NativeMediaCapabilities `json:"nativeMedia"` + Protocol int `json:"protocol"` + Service string `json:"service"` + Port int `json:"port"` + InstanceToken string `json:"instanceToken"` + NativeMedia NativeMediaCapabilities `json:"nativeMedia"` + QualityPreference bool `json:"qualityPreference,omitempty"` } type NativeMediaCapabilities struct { @@ -81,16 +84,17 @@ type NativeMediaCapabilities struct { } type Server struct { - ctx context.Context - cancel context.CancelFunc - listener net.Listener - httpServer *http.Server - endpoint Endpoint - allowedOrigins []string - nativeMedia NativeMediaCapabilities - newControl func() ControlSession - presentation func(string) - done chan error + ctx context.Context + cancel context.CancelFunc + listener net.Listener + httpServer *http.Server + endpoint Endpoint + allowedOrigins []string + nativeMedia NativeMediaCapabilities + newControl func() ControlSession + presentation func(string) + qualityPreference *appconfig.QualityPreference + done chan error mu sync.Mutex connections map[*websocket.Conn]struct{} @@ -129,12 +133,13 @@ func Start(parent context.Context, options Options) (*Server, error) { Port: address.Port, InstanceToken: instanceToken, }, - allowedOrigins: normalizedOrigins(options.AllowedOrigins), - nativeMedia: options.NativeMedia, - newControl: options.NewControl, - presentation: options.Presentation, - done: make(chan error, 1), - connections: make(map[*websocket.Conn]struct{}), + allowedOrigins: normalizedOrigins(options.AllowedOrigins), + nativeMedia: options.NativeMedia, + newControl: options.NewControl, + presentation: options.Presentation, + qualityPreference: options.QualityPreference, + done: make(chan error, 1), + connections: make(map[*websocket.Conn]struct{}), } server.httpServer = &http.Server{ Handler: server, @@ -216,6 +221,8 @@ func (server *Server) ServeHTTP(response http.ResponseWriter, request *http.Requ server.handleControl(response, request) case "/presentation": server.handlePresentation(response, request) + case "/quality-preference": + server.handleQualityPreference(response, request) default: http.NotFound(response, request) } @@ -265,11 +272,12 @@ func (server *Server) handleHealth(response http.ResponseWriter, request *http.R } response.Header().Set("Cache-Control", "no-store") writeJSON(response, http.StatusOK, Health{ - Protocol: ProtocolVersion, - Service: ServiceName, - Port: server.endpoint.Port, - InstanceToken: server.endpoint.InstanceToken, - NativeMedia: server.nativeMedia, + Protocol: ProtocolVersion, + Service: ServiceName, + Port: server.endpoint.Port, + InstanceToken: server.endpoint.InstanceToken, + NativeMedia: server.nativeMedia, + QualityPreference: server.qualityPreference != nil, }) } diff --git a/src/client/native/quality-preference.ts b/src/client/native/quality-preference.ts new file mode 100644 index 000000000..82d186f3a --- /dev/null +++ b/src/client/native/quality-preference.ts @@ -0,0 +1,65 @@ +import { qualitySettingsSchema, type QualitySettings } from "../../shared/protocol"; +import { debugError } from "../lib/debug"; +import { discoverNativeHealth } from "./client"; +import type { NativeHealth } from "./wire"; + +/** Optional App storage; never claims a media control session or delays sharing. */ +export class AppQualityPreference { + private pending: QualitySettings | null = null; + private writing = false; + + private constructor( + private readonly health: NativeHealth, + private readonly signal: AbortSignal, + ) {} + + static async connect(signal: AbortSignal): Promise { + if (signal.aborted) return null; + const health = await discoverNativeHealth(); + return health?.qualityPreference && !signal.aborted + ? new AppQualityPreference(health, signal) : null; + } + + async read(): Promise { + const value: unknown = await (await this.request()).json(); + return value === null ? null : qualitySettingsSchema.parse(value); + } + + save(settings: QualitySettings): void { + if (this.signal.aborted) return; + this.pending = settings; + if (!this.writing) void this.flush(); + } + + private async flush(): Promise { + this.writing = true; + try { + // A slider can change during a write. Keep only its latest applied choice. + while (this.pending && !this.signal.aborted) { + const settings = this.pending; + this.pending = null; + try { + await this.request(settings); + } catch (error) { + if (!this.signal.aborted) debugError("native", "quality-preference-save-failed", error); + } + } + } finally { + this.pending = null; + this.writing = false; + } + } + + private async request(settings?: QualitySettings): Promise { + const response = await fetch(`http://127.0.0.1:${this.health.port}/quality-preference`, { + method: settings ? "PUT" : "GET", + headers: { "Content-Type": "application/json", "X-Piik-Instance": this.health.instanceToken }, + ...(settings ? { body: JSON.stringify(settings) } : {}), + cache: "no-store", + signal: AbortSignal.any([this.signal, AbortSignal.timeout(4_000)]), + targetAddressSpace: "loopback", + } as RequestInit); + if (!response.ok) throw new Error(`Piik App preference request failed (${response.status})`); + return response; + } +} diff --git a/src/client/native/wire.ts b/src/client/native/wire.ts index 8f4542aa6..49db1be37 100644 --- a/src/client/native/wire.ts +++ b/src/client/native/wire.ts @@ -26,6 +26,7 @@ export const nativeDiscoveryIdentitySchema = z.object({ export const nativeHealthSchema = nativeDiscoveryIdentitySchema.extend({ protocol: z.literal(NATIVE_CLIENT_PROTOCOL), + qualityPreference: z.boolean().optional(), instanceToken: z .string() .length(43) diff --git a/src/client/pages/HostPage.tsx b/src/client/pages/HostPage.tsx index 4a3b0ae26..2ba9e932e 100644 --- a/src/client/pages/HostPage.tsx +++ b/src/client/pages/HostPage.tsx @@ -89,6 +89,7 @@ import { type HostCreationProfile, } from "../lib/creation-profile"; import { readPreferredQuality, savePreferredQuality } from "../lib/quality-preference"; +import { AppQualityPreference } from "../native/quality-preference"; import { createOpaqueId } from "../lib/opaque-id"; import { debugError, debugEvent, debugOperation } from "../lib/debug"; import { @@ -611,6 +612,9 @@ export function HostPage({ const qualityChangeRef = useRef(null); const pendingQualityChangeRef = useRef(null); const qualitySettingsRef = useRef(qualitySettings); + const appQualityPreferenceRef = useRef(null); + const qualityPreferenceEditedRef = useRef(false); + const pendingAppQualityPreferenceRef = useRef(null); const routePolicyRef = useRef(routePolicy); const advancedQualityRef = useRef(advancedQuality); const videoCodecModeRef = useRef(videoCodecMode); @@ -666,6 +670,18 @@ export function HostPage({ ? resolvedVideoCodec : videoCodecMode; + useEffect(() => { + if (!launchedByClient) return; + const controller = new AbortController(); + void loadAppQualityPreference(controller.signal).catch(error => { + if (!controller.signal.aborted) debugError("native", "quality-preference-load-failed", error); + }); + return () => { + controller.abort(); + appQualityPreferenceRef.current = null; + }; + }, [launchedByClient]); + useEffect(() => { if (videoRef.current) { videoRef.current.srcObject = stream; @@ -1602,6 +1618,32 @@ export function HostPage({ setAdvancedQuality(visibleSettings); } + function rememberQuality(settings: QualitySettings): void { + const preference = appQualityPreferenceRef.current; + if (preference) preference.save(settings); + else { + pendingAppQualityPreferenceRef.current = settings; + savePreferredQuality(settings); + } + } + + async function loadAppQualityPreference(signal: AbortSignal): Promise { + const initial = qualitySettingsRef.current; + const preference = await AppQualityPreference.connect(signal); + if (!preference || signal.aborted) return; + appQualityPreferenceRef.current = preference; + // An explicit choice made during discovery wins, even before permission resolves. + const pending = pendingAppQualityPreferenceRef.current; + pendingAppQualityPreferenceRef.current = null; + if (pending) preference.save(pending); + const saved = await preference.read(); + if (!signal.aborted && saved && !qualityPreferenceEditedRef.current && + qualitySettingsRef.current === initial && activeGenerationRef.current === null && + roomMutationRef.current === null) { + commitQuality(saved); + } + } + function changeScreenAudioQuality( screenAudioQuality: ScreenAudioQuality, ): void { @@ -1640,6 +1682,7 @@ export function HostPage({ } async function changeQuality(nextProfile: QualitySettings): Promise { + qualityPreferenceEditedRef.current = true; advancedQualityRef.current = nextProfile; setAdvancedQuality(nextProfile); if (phase !== "live") { @@ -1647,7 +1690,7 @@ export function HostPage({ return; } commitQuality(nextProfile); - savePreferredQuality(nextProfile); + rememberQuality(nextProfile); return; } if (qualityChangeRef.current) { @@ -1719,7 +1762,7 @@ export function HostPage({ } commitQuality(appliedProfile); - savePreferredQuality(appliedProfile); + rememberQuality(appliedProfile); debugEvent("quality", "committed", { generation, applied: appliedProfile }); outcome = "applied"; if (nativeUpdate) { diff --git a/tests/host-quality-ownership.test.ts b/tests/host-quality-ownership.test.ts index 1beeadad1..359a3d44f 100644 --- a/tests/host-quality-ownership.test.ts +++ b/tests/host-quality-ownership.test.ts @@ -19,7 +19,7 @@ import { hostActionErrorNotice, hostFailureCode, isCapturePermissionFailure } fr const source = ts.createSourceFile("HostPage.tsx", readFileSync( new URL("../src/client/pages/HostPage.tsx", import.meta.url), "utf8", ), ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX); -const owners = new Set(["changeQuality", "commitQuality", "handleSignalMessage", +const owners = new Set(["changeQuality", "commitQuality", "rememberQuality", "loadAppQualityPreference", "handleSignalMessage", "switchSource", "watchCaptureEnd", "switchNativeSource", "finishSourceSwitch", "replaceBrowserStream", "recoverBrowserFanout", "disposeNativeShare", "acquireNativeClient", "requestSharing", "startNativeShare", "startBrowserNativeIngress", "ownNativeClient", "discardNativeClient", "releaseUnusedNativeClient", "closeCaptureSourcePicker", @@ -91,6 +91,8 @@ function fixture(launchedByClient = true) { NativeMediaBridge: vi.fn(function (_shareId: string, _client: unknown, _onFailed: () => void) { return bridge; }), manualVideoCodecPreference: vi.fn(), phase: "live", qualitySettingsRef: ref(original), advancedQualityRef: ref(original), qualityChangeRef: ref(null), pendingQualityChangeRef: ref(null), + appQualityPreferenceRef: ref<{ save: (quality: QualitySettings) => void } | null>(null), + qualityPreferenceEditedRef: ref(false), pendingAppQualityPreferenceRef: ref(null), activeGenerationRef: ref(1), streamRef: ref(stream), sourceSwitchRef: ref(null), nativeSourceAudioRef: ref(undefined), setMicrophoneEnabled: vi.fn(), microphoneVoiceProcessing: true, setMicrophoneVoiceProcessing: vi.fn(), @@ -424,6 +426,56 @@ describe("Host room-link copy feedback", () => { }); describe("Host quality ownership", () => { + it("restores App preferences without acquiring media or changing a room", async () => { + const current = fixture(); + current.activeGenerationRef.current = null; + const preference = { read: vi.fn(async () => lower), save: vi.fn() }; + current.context.AppQualityPreference = { connect: vi.fn(async () => preference) }; + await current.context.loadAppQualityPreference(new AbortController().signal); + expect(current.qualitySettingsRef.current).toEqual(lower); + expect(current.NativeClient.connect).not.toHaveBeenCalled(); + expect(current.signalRef.current.setHostQualitySettings).not.toHaveBeenCalled(); + expect(preference.save).not.toHaveBeenCalled(); + }); + + it.each(["explicit", "room", "sharing", "creating", "retired"])( + "does not let a late App preference override %s ownership", async (owner) => { + const current = fixture(); + current.activeGenerationRef.current = null; + const pending = deferred(); + const preference = { read: vi.fn(() => pending.promise), save: vi.fn() }; + current.context.AppQualityPreference = { connect: vi.fn(async () => preference) }; + const controller = new AbortController(); + const loading = current.context.loadAppQualityPreference(controller.signal); + await Promise.resolve(); + if (owner === "explicit") current.qualityPreferenceEditedRef.current = true; + if (owner === "room") current.context.commitQuality({ ...original }); + if (owner === "sharing") current.activeGenerationRef.current = 2; + if (owner === "creating") current.roomMutationRef.current = {}; + if (owner === "retired") controller.abort(); + pending.resolve(lower); + await loading; + expect(current.qualitySettingsRef.current).toEqual(original); + expect(preference.save).not.toHaveBeenCalled(); + }, + ); + + it("carries an applied choice across delayed App discovery without persisting room snapshots", async () => { + const current = fixture(); + current.context.phase = "idle"; + current.activeGenerationRef.current = null; + const preference = { read: vi.fn(async () => original), save: vi.fn() }; + current.context.AppQualityPreference = { connect: vi.fn(async () => preference) }; + await current.change(lower); + await current.context.loadAppQualityPreference(new AbortController().signal); + expect(preference.save).toHaveBeenCalledExactlyOnceWith(lower); + expect(current.qualitySettingsRef.current).toEqual(lower); + current.context.commitQuality(original); + expect(preference.save).toHaveBeenCalledOnce(); + await current.change(lower); + expect(preference.save).toHaveBeenCalledTimes(2); + }); + it("remembers an explicit pre-share choice without changing room or capture state", async () => { const current = fixture(); current.context.phase = "idle"; diff --git a/tests/native-quality-preference.test.ts b/tests/native-quality-preference.test.ts new file mode 100644 index 000000000..9e0fc9f33 --- /dev/null +++ b/tests/native-quality-preference.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_QUALITY_SETTINGS } from "../src/shared/protocol"; +import { AppQualityPreference } from "../src/client/native/quality-preference"; +import { discoverNativeHealth } from "../src/client/native/client"; +import { nativeHealthSchema, NATIVE_CLIENT_PROTOCOL, NATIVE_CLIENT_PORT_START } from "../src/client/native/wire"; + +vi.mock("../src/client/native/client", () => ({ discoverNativeHealth: vi.fn() })); +afterEach(() => { vi.unstubAllGlobals(); vi.clearAllMocks(); }); + +const health = nativeHealthSchema.parse({ + protocol: NATIVE_CLIENT_PROTOCOL, service: "piik-client", port: NATIVE_CLIENT_PORT_START, + instanceToken: "a".repeat(43), qualityPreference: true, +}); + +describe("App quality preference transport", () => { + it("leaves older Apps alone and does not discover after retirement", async () => { + const fetcher = vi.fn(); + vi.stubGlobal("fetch", fetcher); + const controller = new AbortController(); + vi.mocked(discoverNativeHealth).mockResolvedValue({ ...health, qualityPreference: undefined }); + expect(await AppQualityPreference.connect(controller.signal)).toBeNull(); + controller.abort(); + expect(await AppQualityPreference.connect(controller.signal)).toBeNull(); + expect(discoverNativeHealth).toHaveBeenCalledOnce(); + expect(fetcher).not.toHaveBeenCalled(); + }); + + it("uses the advertised App instance and bounds writes to the latest applied choice", async () => { + vi.mocked(discoverNativeHealth).mockResolvedValue(health); + let finish!: (value: Response) => void; + const pending = new Promise(resolve => { finish = resolve; }); + const fetcher = vi.fn() + .mockResolvedValueOnce(new Response(JSON.stringify(DEFAULT_QUALITY_SETTINGS))) + .mockReturnValueOnce(pending) + .mockResolvedValue(new Response(null, { status: 204 })); + vi.stubGlobal("fetch", fetcher); + const controller = new AbortController(); + const preference = (await AppQualityPreference.connect(controller.signal))!; + expect(await preference.read()).toEqual(DEFAULT_QUALITY_SETTINGS); + preference.save(DEFAULT_QUALITY_SETTINGS); + preference.save({ ...DEFAULT_QUALITY_SETTINGS, maxBitrate: 4_000_000 }); + const latest = { ...DEFAULT_QUALITY_SETTINGS, maxBitrate: 5_000_000 }; + preference.save(latest); + expect(fetcher).toHaveBeenCalledTimes(2); + finish(new Response(null, { status: 204 })); + await vi.waitFor(() => expect(fetcher).toHaveBeenCalledTimes(3)); + expect(fetcher).toHaveBeenLastCalledWith(`http://127.0.0.1:${health.port}/quality-preference`, expect.objectContaining({ + method: "PUT", body: JSON.stringify(latest), cache: "no-store", + headers: expect.objectContaining({ "X-Piik-Instance": health.instanceToken }), + })); + controller.abort(); + preference.save(DEFAULT_QUALITY_SETTINGS); + expect(fetcher).toHaveBeenCalledTimes(3); + }); + + it("rejects invalid replies and treats failed writes as optional", async () => { + vi.mocked(discoverNativeHealth).mockResolvedValue(health); + const fetcher = vi.fn().mockResolvedValueOnce(new Response("{}")) + .mockResolvedValueOnce(new Response(null, { status: 500 })) + .mockResolvedValue(new Response(null, { status: 204 })); + vi.stubGlobal("fetch", fetcher); + const preference = (await AppQualityPreference.connect(new AbortController().signal))!; + await expect(preference.read()).rejects.toThrow(); + preference.save(DEFAULT_QUALITY_SETTINGS); + preference.save({ ...DEFAULT_QUALITY_SETTINGS, maxBitrate: 4_000_000 }); + await vi.waitFor(() => expect(fetcher).toHaveBeenCalledTimes(3)); + }); +}); From bf7213ca16c631732367e3816300375d858284b1 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:27:08 +0800 Subject: [PATCH 010/262] test(media): distinguish source cadence from SFU readiness --- docs/operations/website.md | 15 ++++++++++ docs/research/nat-traversal.md | 2 +- docs/research/native-client-media.md | 29 +++++++++++++++++++ docs/todo.md | 42 +++++++++++----------------- scripts/embedded-sfu-gate.ts | 20 ++++++++++++- scripts/embedded-sfu-page.ts | 2 +- 6 files changed, 82 insertions(+), 28 deletions(-) diff --git a/docs/operations/website.md b/docs/operations/website.md index 61fae6237..9899e2951 100644 --- a/docs/operations/website.md +++ b/docs/operations/website.md @@ -24,6 +24,21 @@ GitHub Pages can serve the complete website: it hosts The demo also needs the Go service for room authority, WebSocket signaling and UDP STUN, so it runs on the separate US server rather than Pages. +### Managed Hosting Assessment + +Reviewed 2026-10-07. Keep the current deployment for this candidate. A managed +container can reduce host administration, but must preserve public UDP, stable +room authority and SQLite storage. [Fly.io supports public UDP](https://docs.fly.io/networking/udp-and-tcp) +with a dedicated IPv4 address and a specific UDP bind address; external and +internal UDP ports must match. Its [dedicated IPv4 charge](https://docs.fly.io/about/pricing) +is $2/month before compute, storage and transfer. This is a possible deployment +target, not an established cost saving over the current server. + +[Cloudflare Containers disks are ephemeral by default](https://developers.cloudflare.com/containers/faq/#is-disk-persistent-what-happens-to-my-disk-when-my-container-sleeps). +Snapshots or object-backed storage are not a drop-in replacement for the current +live SQLite contract. No provider migration is accepted; compare the complete +cost and verify storage, UDP and restart behavior before changing the Demo. + ## Public Demo Configuration The chosen demo allows entry without a site password and provides P2P sharing diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index cee29a1ae..d7f6e1e5a 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -353,7 +353,7 @@ increasing only the fixture's ICE budget recovered media. The ordinary Pion loss control recovered without that override. Do not adopt packet prefixes, diagnostic Ping polling or wider product deadlines as integration shortcuts. -Current recommendation: keep the transport isolated. Go reuse is feasible, but +Reassessed on 2026-10-07: keep the transport isolated. Go reuse is feasible, but new key authorization/revocation, underlay readiness, packet-size limits and relay operation have no demonstrated direct-coverage benefit sufficient for product integration. A future trial needs matched networks with a repeatable diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 8da28b7e4..708e5f0a8 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -357,6 +357,25 @@ that result nor the allocation timings establish the reported system-wide lag's cause. Retain the existing implementation; reopen optimization from a matched bottleneck. +### Virtual-Display Acceptance Limit + +On 2026-10-07, the local Native H.264 SFU gate initially passed its two-room +check, then missed its unchanged 300-frame/30-second requirement. The repeat +environment exposed only a GameViewer virtual display. Current main reproduced +the failure with its own App and capture binary: video/audio arrived, but video +was roughly 4–7 fps without receiver loss or decode drops. A source-window draw +counter advanced about 133 times per second; that counts drawing attempts, not +Windows capture delivery. + +Temporary capture instrumentation observed about 3–4 WGC frame-arrival events +per second. Queue draining was bounded in the observation, conversion/submission +took less than a millisecond, and encoder diagnostics showed small encode times +without CPU/quality adaptation. Bringing the source window forward did not +restore cadence. This localizes the limit before encoding but does not establish +the virtual driver's cause or diagnose a field report. Retain the gate's +threshold and compare a physical display before changing capture scheduling, +encoder policy or route behavior. + ## Native H.264 Motion Quality Issue [#432](https://github.com/TNTcraftHIM/Piik/issues/432) reports more visible @@ -534,6 +553,16 @@ inherently require a major; breaking an existing public contract does. Evaluate the Browser/App/Server and mixed-peer flows before freezing a replacement contract; avoid a general compatibility framework. +The 2026-10-07 candidate review does not recommend adding these extensions to +the current release. A 4K path must change the three platform capture/decoder +bounds, encoded-frame limits and SFU admission together; Windows H.264's current +level ceiling also does not cover 4K60. The existing Main/High comparison shows +a modest synthetic quality gain, but Firefox's receiver gap prevents treating +it as a transparent upgrade. Keep the current formats and limits until a +specific use case justifies that coordinated change. This is a scope +recommendation, not a claim that 4K is impossible or that the current geometry +should become a permanent protocol rule. + ## Implementation Boundary - `nativecapture` owns the child process, source identity, and bounded frame protocol. diff --git a/docs/todo.md b/docs/todo.md index 0d40b53b4..36ad1f19d 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -12,29 +12,12 @@ history. A parked idea is not implementation authority. coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. -- [ ] **App-wide sharing preferences (#460).** Same-origin Browser preferences - are implemented. Assess retaining explicit settings across changing App public - origins without eagerly claiming Native sessions, adding a second settings - owner or persisting live device/session authority. Keep the Browser scope clear - if an App-wide bridge is not justified. -- [ ] **Current issue evidence.** Review #452's public-link/H.264 report and - #453's cross-network failure against current startup and ICE behavior. Revisit - #432's motion-quality evidence and existing unmatched reports; do not equate a - successful local reproduction with resolving a different reporter's cause. - [ ] **Current-contract runtime acceptance.** Exercise sharing, source/quality replacement, reconnect, peer/SFU handoff and independent retirement. Review responsive interaction and reader documentation against the accepted result; - keep unavailable physical-device checks explicit. -- [ ] **Deferred-work decisions.** Reassess transport, microphone pairing, - passive App attachment, hosting and remaining structural proposals in order - of demonstrated value. Record accepted scope in the owning modules before - implementation; do not turn every proposal into a release requirement. -- [ ] **4K, codec expansion and Native Magicsock assessment.** Reopened by the - owner on 2026-10-07 for 2.0 only where measured benefit justifies a simple - implementation. Review the [media extension assessment](./research/native-client-media.md#media-capability-extension-assessment) - and [transport comparison](./research/nat-traversal.md#integration-cost-and-recovery-controls) - against current main. No changed wire contract or product transport is accepted - merely by reopening the experiments. + keep unavailable physical-device checks explicit. The local Native SFU + throughput rerun is limited by [low WGC arrival cadence](./research/native-client-media.md#virtual-display-acceptance-limit) + reproduced on unchanged main; do not lower its frame requirement to report a pass. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance are complete. Integrate through the next owner-approved publishing merge and verify the separate workflow with its original CI artifacts. Mirror failure @@ -58,12 +41,19 @@ history. A parked idea is not implementation authority. ## Deferred Feature Work -These proposals remain deferred beyond the accepted interaction phase. +These proposals remain outside the current candidate. Reassessment on +2026-10-07 did not establish enough benefit for their added owners and operating +cost. The media and transport assessments below remain recommendations for the +owner's final scope review, not permanent prohibitions. -- [ ] **Managed Demo hosting.** Evaluate cost and public UDP support before any - migration. The assessment remains outside the current product release; keep - the existing P2P-only Demo deployment until a provider and operating boundary - are accepted. +- [ ] **4K and additional codecs.** The [media assessment](./research/native-client-media.md#media-capability-extension-assessment) + recommends retaining current limits for this candidate. Reopen with a concrete + source/receiver use case that justifies coordinated capture, forwarding and + resource bounds; keep UI presets separate from those bounds in any new design. + +- [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) + identifies UDP/storage requirements but establishes no cost saving. Keep the + existing P2P-only Demo until a provider and operating boundary are accepted. - [ ] **Phone as a Host microphone: assess after the current work.** Explore an opt-in link that pairs a phone's microphone with the Host's existing audio mixer. Evaluate pairing/revocation, latency/echo, browser background limits @@ -108,6 +98,8 @@ of a reporter's cause; research owns the completed experiments and their limits. [#443](https://github.com/TNTcraftHIM/Piik/issues/443) lacks a connected Native edge and its paired Viewer/App transport report; [#448](https://github.com/TNTcraftHIM/Piik/issues/448) only reaches route allocation. + [#453](https://github.com/TNTcraftHIM/Piik/issues/453) has no matched endpoint + reports to distinguish network reachability from a product failure. The v1.6.3 candidate-timeout report lacks share-start and SDP/ICE evidence. - [ ] **Camera and Host microphone device coverage.** Check real audio levels, echo, device replacement/native mixing and phone camera permission, orientation diff --git a/scripts/embedded-sfu-gate.ts b/scripts/embedded-sfu-gate.ts index b18e8bd7a..47b7c5b4f 100644 --- a/scripts/embedded-sfu-gate.ts +++ b/scripts/embedded-sfu-gate.ts @@ -104,6 +104,7 @@ async function main(): Promise { host: null as Snapshot | null, viewer: null as Snapshot | null, displayLifecycle: null as { hiddenFrames: number; hiddenElapsedMs: number; lowLayerDecoded: boolean; resumed: boolean; replaced: boolean } | null, mediaDiagnostics: null as unknown, + sourceDrawing: null as { draws: number; elapsedMs: number; visibility: string } | null, publicationRetired: false, subscriptionRecovered: false, nativeShareStopped: !nativeArm, udpReleased: false, cleanup: false, error: null as string | null }; let server: ChildProcessWithoutNullStreams | null = null; let chrome: ChildProcessWithoutNullStreams | null = null; @@ -118,6 +119,7 @@ async function main(): Promise { let nativePort = 0; let sourceChrome: ChildProcessWithoutNullStreams | null = null; let sourceCdp: CdpConnection | null = null; + let sourcePage: PageHandle | null = null; let sourceHTTP: Awaited> | null = null; const call = async (page: PageHandle, expression: string, timeout = 25_000): Promise => { const value = await evaluate<{ value?: T; error?: string }>(cdp!, page, @@ -195,6 +197,7 @@ async function main(): Promise { if (!target) throw new Error("Gate source page is unavailable"); const attached = await sourceCdp.call<{ sessionId: string }>("Target.attachToTarget", { targetId: target.targetId, flatten: true }, undefined, Date.now() + 5_000); + sourcePage = attached; const audioState = await evaluate(sourceCdp, attached, `(async () => { const audio = new AudioContext(), tone = audio.createOscillator(), gain = audio.createGain(); gain.gain.value = 0.01; tone.connect(gain).connect(audio.destination); tone.start(); @@ -256,7 +259,21 @@ async function main(): Promise { result.stage = "simulcast-first-frame"; await cdp.call("Page.bringToFront", {}, viewer.sessionId, Date.now() + 3_000); result.host = await call(host, "gate.snapshot()"); - result.high = await call(viewer, nativeArm ? "gate.waitForFrames(1280, 720, 300)" : "gate.waitForFrames(1280, 720)", nativeArm ? 32_000 : 25_000); + const sourceSample = async () => sourceCdp && sourcePage + ? await evaluate<{ draws: number; at: number; visibility: string }>(sourceCdp, sourcePage, + "({ draws: n, at: performance.now(), visibility: document.visibilityState })", Date.now() + 3_000) + : null; + const sourceBefore = await sourceSample(); + try { + result.high = await call(viewer, nativeArm ? "gate.waitForFrames(1280, 720, 300)" : "gate.waitForFrames(1280, 720)", nativeArm ? 32_000 : 25_000); + } finally { + const sourceAfter = await sourceSample().catch(() => null); + if (sourceBefore && sourceAfter) result.sourceDrawing = { + draws: sourceAfter.draws - sourceBefore.draws, + elapsedMs: sourceAfter.at - sourceBefore.at, + visibility: sourceAfter.visibility, + }; + } result.stage = "decoded-audio-energy"; await waitForSample(() => call(viewer!, "gate.audioEnergy()", 3_000), (energy) => { result.audioEnergy.high = energy; return energy > 0; }, 5_000); @@ -337,6 +354,7 @@ async function main(): Promise { if (cdp) result.mediaDiagnostics = await Promise.all([host, viewer].map(page => page ? evaluate(cdp!, page, "window.__piikGateMediaStats?.()", Date.now() + 3_000).catch(() => null) : null)); if (viewer && cdp) result.viewer = await call(viewer, "gate.snapshot()", 3_000).catch(() => null); + if (host && cdp) result.host = await call(host, "gate.snapshot()", 3_000).catch(() => null); if (viewer && cdp && result.stage === "decoded-audio-energy") result.audioDiagnostics = await evaluate( cdp, viewer, "window.__piikGateAudioStats ?? null", Date.now() + 3_000).catch(() => null); if (serverError) result.error += `; server: ${serverError}`; diff --git a/scripts/embedded-sfu-page.ts b/scripts/embedded-sfu-page.ts index 207de8ebb..a6d588119 100644 --- a/scripts/embedded-sfu-page.ts +++ b/scripts/embedded-sfu-page.ts @@ -337,7 +337,7 @@ export async function waitForFrames(width: number, height: number, minimumFrames let callback = 0; const timer = setTimeout(() => { target.cancelVideoFrameCallback(callback); - reject(new Error(`Viewer did not decode ${width}x${height}; current ${target.videoWidth}x${target.videoHeight}; ${frames} frame callbacks; ${document.visibilityState}; ${error ?? "no signaling error"}`)); + reject(new Error(`Viewer did not reach ${minimumFrames} decoded frame callbacks at ${width}x${height}; current ${target.videoWidth}x${target.videoHeight}; ${frames} matching callbacks; ${document.visibilityState}; ${error ?? "no signaling error"}`)); }, minimumFrames > 15 ? 30_000 : 20_000); const next = () => { callback = target.requestVideoFrameCallback(() => { From 56593285d597d1608076e2e5d13de665b6a6c912 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:43:17 +0800 Subject: [PATCH 011/262] docs(media): bound the local capture cadence finding --- docs/research/native-client-media.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 708e5f0a8..cc46f0a61 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -371,7 +371,9 @@ Temporary capture instrumentation observed about 3–4 WGC frame-arrival events per second. Queue draining was bounded in the observation, conversion/submission took less than a millisecond, and encoder diagnostics showed small encode times without CPU/quality adaptation. Bringing the source window forward did not -restore cadence. This localizes the limit before encoding but does not establish +restore cadence. A capture-only control, without the App or SFU, also delivered +about 3.9 fps from both Chrome and a separate animated Windows Forms window. +This localizes the limit before encoding but does not establish the virtual driver's cause or diagnose a field report. Retain the gate's threshold and compare a physical display before changing capture scheduling, encoder policy or route behavior. From 921196bfc6ddd459819bdf0d019c8b9f3d0c7127 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:10:43 +0800 Subject: [PATCH 012/262] test(media): use public invitations for remote acceptance --- cmd/piik-app/README.md | 23 +++------- cmd/piik-app/README.zh-CN.md | 17 ++------ cmd/piik-peer-gate/main.go | 9 +++- cmd/piik-peer-gate/main_test.go | 44 +++++++++++++++++++ docs/research/native-client-media.md | 10 +++-- internal/app/app.go | 5 +-- internal/app/app_test.go | 4 +- scripts/app-native-host-gate.ts | 65 ++++------------------------ 8 files changed, 81 insertions(+), 96 deletions(-) diff --git a/cmd/piik-app/README.md b/cmd/piik-app/README.md index 7e487a5a2..ad5bb8ec9 100644 --- a/cmd/piik-app/README.md +++ b/cmd/piik-app/README.md @@ -258,16 +258,6 @@ CHROME_PATH=/path/to/chrome \ PIIK_GO=/path/to/go \ npm run gate:app-native-host -PIIK_CLIENT_NATIVE_HOST_GATE=true \ -PIIK_CLIENT_CROSS_NAT_GATE=true \ -CHROME_PATH=/path/to/chrome \ -PIIK_GO=/path/to/go \ -PIIK_REMOTE_HOST= \ -PIIK_REMOTE_USER= \ -PIIK_REMOTE_SSH_KEY=/path/to/key \ -PIIK_CLIENT_GATE_STUN_URLS=stun::3478 \ -npm run gate:app-native-host - PIIK_CLIENT_NATIVE_HOST_GATE=true \ PIIK_CLIENT_LINK_MEDIA_GATE=true \ CHROME_PATH=/path/to/chrome \ @@ -291,11 +281,12 @@ The Windows media gate proves one hardware-H.264 capture generation, shared Pion source, Browser decode, PLI recovery, and STUN candidate gathering. The native Host gate proves room creation and native video delivery through the current route; native audio is included when the capability -probe and target OS support it. The cross-NAT variant uses a temporary reverse -SSH path for signaling only and requires a selected `srflx` or `prflx` media pair; -media never travels through SSH. The one-link media variant instead carries the -same signaling through the App's temporary public origin and requires direct -media delivery to an independent Linux peer. Native P2P quality evidence and -embedded SFU delivery have explicit gates. macOS and Linux capture still +probe and target OS support it. The one-link media variant carries signaling +through the App's temporary public origin and requires direct video RTP delivery +to an independent Linux peer, with a selected `srflx` or `prflx` candidate. +SSH only runs the remote probe; media never travels through it. Without a remote +host, this variant checks decoded media in a local Browser through the public +invitation; that does not establish cross-network connectivity. Native P2P +quality evidence and embedded SFU delivery have explicit gates. macOS and Linux capture still require physical desktop/media gates; CI compilation and package smoke do not substitute for them. diff --git a/cmd/piik-app/README.zh-CN.md b/cmd/piik-app/README.zh-CN.md index 8555b0fa5..a10268846 100644 --- a/cmd/piik-app/README.zh-CN.md +++ b/cmd/piik-app/README.zh-CN.md @@ -213,16 +213,6 @@ CHROME_PATH=/path/to/chrome \ PIIK_GO=/path/to/go \ npm run gate:app-native-host -PIIK_CLIENT_NATIVE_HOST_GATE=true \ -PIIK_CLIENT_CROSS_NAT_GATE=true \ -CHROME_PATH=/path/to/chrome \ -PIIK_GO=/path/to/go \ -PIIK_REMOTE_HOST= \ -PIIK_REMOTE_USER= \ -PIIK_REMOTE_SSH_KEY=/path/to/key \ -PIIK_CLIENT_GATE_STUN_URLS=stun::3478 \ -npm run gate:app-native-host - PIIK_CLIENT_NATIVE_HOST_GATE=true \ PIIK_CLIENT_LINK_MEDIA_GATE=true \ CHROME_PATH=/path/to/chrome \ @@ -246,7 +236,8 @@ Windows media gate 验证一个硬件 H.264 采集代次、共享 Pion 来源、 STUN 候选收集。native Host gate 验证房间创建及当前路由上的原生视频传输; 能力探测与目标系统支持时,也包含原生音频。 -跨 NAT 变体仅将临时反向 SSH 路径用于信令,并要求选中的媒体候选对包含 `srflx` 或 `prflx`; -媒体不会经过 SSH。公网邀请媒体变体通过 App 临时公网来源传送相同信令, -并要求将媒体直接传给独立的 Linux 对端。原生 P2P 质量证据与内嵌 SFU 传输有各自的验证入口。 +公网邀请媒体变体通过 App 临时公网地址传送信令,要求将视频 RTP 直接传给独立的 Linux 对端, +且选中的候选对包含 `srflx` 或 `prflx`。SSH 仅运行远端探针,媒体不会经过 SSH。 +不配置远端主机时,该变体在本机浏览器中通过公网邀请检查解码画面,不能据此认定跨网络连通。 +原生 P2P 质量证据与内嵌 SFU 传输有各自的验证入口。 macOS 和 Linux 采集仍需实机桌面与媒体验证,CI 编译及程序包冒烟检查不能替代这些验证。 diff --git a/cmd/piik-peer-gate/main.go b/cmd/piik-peer-gate/main.go index 50d2065f7..85a7c56c5 100644 --- a/cmd/piik-peer-gate/main.go +++ b/cmd/piik-peer-gate/main.go @@ -68,6 +68,7 @@ type incoming struct { Revision int `json:"revision"` ConnectionID string `json:"connectionId"` Phase string `json:"phase"` + State string `json:"state"` Candidate struct { ConnectionID string `json:"connectionId"` } `json:"candidate"` @@ -316,7 +317,7 @@ func run(ctx context.Context, config gateConfig) gateResult { return gateResult{Error: "receiver signaling failed"} } case "route-status": - if message.Phase == "failed" { + if message.State == "failed" { return gateResult{Error: "route assignment failed"} } } @@ -384,10 +385,14 @@ func (receiver *receiver) createPeer(raw []wireICEServer) error { pc.OnTrack(func(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { receiver.recordSelectedPair(rtpReceiver) for { - _, _, readErr := track.ReadRTP() + packet, _, readErr := track.ReadRTP() if readErr != nil { return } + // Audio or padding alone cannot prove video delivery. + if track.Kind() != webrtc.RTPCodecTypeVideo || len(packet.Payload) == 0 { + continue + } receiver.packetMu.Lock() receiver.packets++ count := receiver.packets diff --git a/cmd/piik-peer-gate/main_test.go b/cmd/piik-peer-gate/main_test.go index 26e4e3997..a7d66e861 100644 --- a/cmd/piik-peer-gate/main_test.go +++ b/cmd/piik-peer-gate/main_test.go @@ -1,10 +1,54 @@ package main import ( + "context" "encoding/json" + "net/http" + "net/http/httptest" + "strings" "testing" + "time" + + "github.com/TNTcraftHIM/Piik/internal/server/protocol" + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" ) +func TestRunReportsRouteFailure(t *testing.T) { + done := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.CloseNow() + var auth json.RawMessage + if err := wsjson.Read(r.Context(), conn, &auth); err != nil { + t.Error(err) + return + } + if err := wsjson.Write(r.Context(), conn, protocol.RouteStatusMessage{ + Type: "route-status", Revision: 1, State: "failed", Reason: "route-exhausted", + }); err != nil { + t.Error(err) + return + } + <-done + })) + defer server.Close() + defer close(done) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + result := run(ctx, gateConfig{ + SignalURL: "ws" + strings.TrimPrefix(server.URL, "http"), + Origin: server.URL, RoomID: "1234", ClientID: "remote_test", ViewerGrant: "test", + }) + if result.Passed || result.Error != "route assignment failed" { + t.Fatalf("expected the route failure, got %+v", result) + } +} + func TestIncomingUsesCurrentWireFieldNames(t *testing.T) { payload := []byte(`{ "type":"route-update", diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index cc46f0a61..274c441dd 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -373,9 +373,13 @@ took less than a millisecond, and encoder diagnostics showed small encode times without CPU/quality adaptation. Bringing the source window forward did not restore cadence. A capture-only control, without the App or SFU, also delivered about 3.9 fps from both Chrome and a separate animated Windows Forms window. -This localizes the limit before encoding but does not establish -the virtual driver's cause or diagnose a field report. Retain the gate's -threshold and compare a physical display before changing capture scheduling, +A separate minimal WGC program, using only Windows APIs and no Piik code, +reproduced about 3.8 fps while that Forms window painted about 32 fps. +Default, 4 ms and 1 ms capture intervals returned the same rate, including with +the source kept topmost; property readback confirmed each requested interval. +This localizes the limit to platform capture delivery in this environment but +does not establish the virtual driver's cause or diagnose a field report. +Retain the gate's threshold and compare a physical display before changing capture scheduling, encoder policy or route behavior. ## Native H.264 Motion Quality diff --git a/internal/app/app.go b/internal/app/app.go index c70859f3b..9535d5b07 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -641,9 +641,8 @@ func discoverNativeMedia(ctx context.Context, configuredPath string) nativeRunti // localSTUNURLs is the Local room authority's ICE configuration. Only --link // has a public path, so only --link configures public STUN and the bounded NAT -// prediction survey. The lists are literals on purpose: a gate that sets -// STUN_URLS is configuring the App's own Pion edge, and that value must not -// reach the room server (which is why serverconfig.Local reads no environment). +// prediction survey. Native edges receive ICE configuration from that authority; +// serverconfig.Local deliberately does not inherit Hosted environment settings. func localSTUNURLs(link bool) (stunURLs []string, natPredictionSTUNURLs []string) { if !link { return nil, nil diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 2b17851cc..885abf1a4 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -699,8 +699,8 @@ func TestLinkModeKeepsOneLocalAuthority(t *testing.T) { } } -// The Local room authority owns its ICE configuration: a STUN_URLS meant for -// the App's own Pion edge must not reach it, and only --link is public. +// The Local room authority owns its ICE configuration independently of Hosted +// environment settings, and only --link is public. func TestLocalServerOwnsItsSTUNConfiguration(t *testing.T) { t.Setenv("STUN_URLS", "stun:inherited.example:3478") for _, testCase := range []struct { diff --git a/scripts/app-native-host-gate.ts b/scripts/app-native-host-gate.ts index 33b752c5e..9683c6975 100644 --- a/scripts/app-native-host-gate.ts +++ b/scripts/app-native-host-gate.ts @@ -31,7 +31,7 @@ const ROOT = resolve(import.meta.dirname, ".."); const BUILD_ROOT = join(ROOT, "build", "go-check"); export const SOURCE_TITLE = "Piik Native Gate Source"; -type GateMode = "local" | "cross-nat" | "one-link"; +type GateMode = "local" | "one-link"; type VideoCodec = "h264" | "vp8"; const CODEC_PROBE = `(() => { @@ -92,7 +92,6 @@ interface GateResult { remoteRemoteCandidateType: string; remoteNatPath: boolean; remotePeerExited: boolean | null; - reverseSignalTunnelClosed: boolean | null; sourceFailureEndedShare: boolean | null; replacementViewerConnected: boolean | null; replacementViewerFrames: number | null; @@ -110,7 +109,6 @@ interface RemoteGateOptions { host: string; user: string; key: string; - signalPort: number; bindAddress: string | null; } @@ -273,18 +271,15 @@ function remoteOptions(): RemoteGateOptions { const host = process.env.PIIK_REMOTE_HOST?.trim(); const user = process.env.PIIK_REMOTE_USER?.trim() || "root"; const key = process.env.PIIK_REMOTE_SSH_KEY?.trim(); - const signalPortText = process.env.PIIK_REMOTE_SIGNAL_PORT?.trim() || "49721"; - const signalPort = Number(signalPortText); - if (!host || !key || !Number.isInteger(signalPort) || signalPort < 1024 || signalPort > 65_535) { + if (!host || !key) { throw new Error( - "PIIK_REMOTE_HOST, PIIK_REMOTE_SSH_KEY, and a valid PIIK_REMOTE_SIGNAL_PORT are required", + "PIIK_REMOTE_HOST and PIIK_REMOTE_SSH_KEY are required", ); } return { host, user, key, - signalPort, bindAddress: process.env.PIIK_REMOTE_BIND_ADDRESS?.trim() || null, }; } @@ -574,16 +569,8 @@ async function main(): Promise { if (process.env.PIIK_CLIENT_NATIVE_HOST_GATE !== "true") { throw new Error("PIIK_CLIENT_NATIVE_HOST_GATE=true is required"); } - const crossNat = process.env.PIIK_CLIENT_CROSS_NAT_GATE === "true"; const linkMedia = process.env.PIIK_CLIENT_LINK_MEDIA_GATE === "true"; - if (crossNat && linkMedia) { - throw new Error("Cross-NAT and one-link gate modes are mutually exclusive"); - } - const mode: GateMode = linkMedia ? "one-link" : crossNat ? "cross-nat" : "local"; - const gateStunUrls = process.env.PIIK_CLIENT_GATE_STUN_URLS?.trim(); - if (mode === "cross-nat" && !gateStunUrls) { - throw new Error("PIIK_CLIENT_GATE_STUN_URLS is required for the cross-NAT gate"); - } + const mode: GateMode = linkMedia ? "one-link" : "local"; const crashGate = process.env.PIIK_CLIENT_NATIVE_HOST_CRASH_GATE === "true"; if (crashGate && mode !== "local") { @@ -597,7 +584,7 @@ async function main(): Promise { if (requestedCodec !== "auto" && requestedCodec !== "h264" && requestedCodec !== "vp8") { throw new Error("PIIK_CLIENT_NATIVE_HOST_CODEC must be auto, h264, or vp8"); } - const remote = mode === "cross-nat" || mode === "one-link" && + const remote = mode === "one-link" && Boolean(process.env.PIIK_REMOTE_HOST?.trim() || process.env.PIIK_REMOTE_SSH_KEY?.trim()) ? remoteOptions() : null; const chromePath = process.env.CHROME_PATH?.trim(); @@ -627,7 +614,6 @@ async function main(): Promise { let chrome: ChildProcessWithoutNullStreams | null = null; let cdp: CdpConnection | null = null; let nativePort = 0; - let remoteTunnel: ChildProcess | null = null; let stage = "setup"; const result: GateResult = { passed: false, @@ -656,7 +642,6 @@ async function main(): Promise { remoteRemoteCandidateType: "", remoteNatPath: false, remotePeerExited: null, - reverseSignalTunnelClosed: mode === "cross-nat" ? false : null, sourceFailureEndedShare: mode === "local" ? false : null, replacementViewerConnected: mode === "local" ? false : null, replacementViewerFrames: mode === "local" ? 0 : null, @@ -727,37 +712,12 @@ async function main(): Promise { ...process.env, PIIK_DEBUG: "", PIIK_CLIENT_GATE_NO_BROWSER: "true", - // STUN_URLS reaches only the App's own Pion edge: the in-process - // room server never reads it, so the cross-NAT arm stays isolated. - ...(mode === "cross-nat" && gateStunUrls - ? { STUN_URLS: gateStunUrls } - : {}), }, }); app.stderr.resume(); stage = "app-ready"; const appInfo = await readAppEndpoint(app, mode === "one-link"); nativePort = appInfo.endpoint.port; - if (mode === "cross-nat" && remote) { - stage = "signaling-tunnel"; - const tunnel = spawn( - process.env.PIIK_SSH?.trim() || "ssh", - [ - "-N", "-T", - ...remoteTransportOptions(remote), - "-o", "ExitOnForwardFailure=yes", - "-R", `127.0.0.1:${remote.signalPort}:127.0.0.1:${appPort}`, - `${remote.user}@${remote.host}`, - ], - { stdio: "ignore", windowsHide: true }, - ); - remoteTunnel = tunnel; - tunnel.on("error", () => undefined); - await new Promise((resolveTunnel) => setTimeout(resolveTunnel, 300)); - if (tunnel.exitCode !== null) { - throw new Error("signaling tunnel could not start"); - } - } stage = "local-server-ready"; await waitForValue( async () => { @@ -893,19 +853,14 @@ async function main(): Promise { } if (remote) { stage = "remote-viewer"; - const signalUrl = mode === "one-link" - ? new URL("/signal", appInfo.publicOrigin!).toString().replace(/^http/, "ws") - : `ws://127.0.0.1:${remote.signalPort}/signal`; - const signalOrigin = mode === "one-link" - ? appInfo.publicOrigin! - : `http://localhost:${appPort}`; + const signalUrl = new URL("/signal", appInfo.publicOrigin!).toString().replace(/^http/, "ws"); const remoteResult = await runRemotePeerGate( remoteBinary, remote, signalUrl, roomMatch[1], viewerGrant, - signalOrigin, + appInfo.publicOrigin!, ); result.remoteViewerConnected = remoteResult.passed; result.remoteViewerPackets = remoteResult.packets; @@ -1467,9 +1422,6 @@ async function main(): Promise { result.error = error instanceof Error ? error.message : String(error); result.stage = stage; } finally { - if (remoteTunnel) { - result.reverseSignalTunnelClosed = await stopChild(remoteTunnel); - } if (sourceChrome && sourceCdp) { await closeSourceBrowser(sourceChrome, sourceCdp); sourceChrome = null; @@ -1505,8 +1457,7 @@ async function main(): Promise { result.hostInvite && (remote ? result.remoteViewerConnected && result.remoteViewerPackets >= 30 && - result.remoteNatPath && result.remotePeerExited === true && - (mode !== "cross-nat" || result.reverseSignalTunnelClosed === true) + result.remoteNatPath && result.remotePeerExited === true : mode === "one-link" ? result.publicViewerPage && result.publicViewerSignal && result.viewerConnected && result.viewerFrames >= 30 && result.viewerWidth === 1920 && result.viewerHeight === 1080 && result.codecPreserved From bc7485bb80670b2f4bcafb8057676de68966f3f8 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:32:43 +0800 Subject: [PATCH 013/262] docs: narrow remaining candidate acceptance --- docs/todo.md | 11 +++++------ scripts/peer-assisted-benchmark.ts | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/todo.md b/docs/todo.md index 36ad1f19d..4957e481c 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -12,12 +12,11 @@ history. A parked idea is not implementation authority. coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. -- [ ] **Current-contract runtime acceptance.** Exercise sharing, source/quality - replacement, reconnect, peer/SFU handoff and independent retirement. Review - responsive interaction and reader documentation against the accepted result; - keep unavailable physical-device checks explicit. The local Native SFU - throughput rerun is limited by [low WGC arrival cadence](./research/native-client-media.md#virtual-display-acceptance-limit) - reproduced on unchanged main; do not lower its frame requirement to report a pass. +- [ ] **Native capture cadence acceptance.** The local Native SFU throughput + rerun is limited by [low WGC arrival cadence](./research/native-client-media.md#virtual-display-acceptance-limit) + reproduced on unchanged main and a separate Windows-API-only capture probe. + Compare on a physical display before claiming current Native throughput + acceptance; do not lower the unchanged 300-frame/30-second requirement. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance are complete. Integrate through the next owner-approved publishing merge and verify the separate workflow with its original CI artifacts. Mirror failure diff --git a/scripts/peer-assisted-benchmark.ts b/scripts/peer-assisted-benchmark.ts index 3348efb39..2d951d313 100644 --- a/scripts/peer-assisted-benchmark.ts +++ b/scripts/peer-assisted-benchmark.ts @@ -3536,7 +3536,7 @@ async function main(): Promise { "CDP SystemInfo exposes no resident-set field, so peakResidentSetBytes is null; GPU, NIC, glass-to-glass latency, generational visual quality, mobile browsers, and SFU require other measurement.", externalServer ? "The external server target is exercised as configured; the gate does not infer unavailable transports." - : "The local runner does not start LiveKit; SFU consistency is reported only when an SFU route is actually observed.", + : "The local runner does not enable embedded SFU; SFU consistency is reported only when an SFU route is actually observed.", "The harness emits raw gate fields and simple invariants; it does not implement a route score or runtime policy.", "BENCHMARK_QUALITY_SMOKE disables encoded-stream support to verify ordinary-sender parameter propagation. Browser pool producer/clock settings are checked by browser-local-pool-probe.ts.", "This gate measures initial topology and relay-loss recovery, not sender-quality convergence or synthetic Viewer-triggered handoffs.", From bf16776a8c4de88c53baabe26960e911c2cdc004 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:18:44 +0800 Subject: [PATCH 014/262] test(server): verify storage failure and cross-room isolation boundaries --- docs/research/server-consolidation.md | 28 +++--- docs/todo.md | 98 +++++++++++++-------- internal/server/room/database_test.go | 120 ++++++++++++++++++++++++++ internal/server/signal/server_test.go | 65 ++++++++++++++ 4 files changed, 260 insertions(+), 51 deletions(-) diff --git a/docs/research/server-consolidation.md b/docs/research/server-consolidation.md index 54c1c855a..167c1c9f0 100644 --- a/docs/research/server-consolidation.md +++ b/docs/research/server-consolidation.md @@ -1,8 +1,9 @@ # Server Consolidation -- Reviewed: 2026-09-06 -- Status: implemented in the candidate; the acceptance list below is the - remaining boundary +- Reviewed: 2026-10-07 +- Status: published; current physical limits are in + [verification status](../verification-status.md). The before/after measurements + below describe the original 2026-09-06 consolidation, not current package cost. - Scope: one shared Go backend for Hosted and self-contained App operation ## Current Ownership @@ -13,9 +14,10 @@ reachability, and lifecycle policy. Within that scope `protocol` owns wire types strict decoding, and shared scalars; `config` owns environment validation plus the Hosted and Local compositions; `room` owns room authority, participants, and the SQLite database; `route` decides graph transitions without I/O; `signal` -executes signaling and SFU effects under one lock; `sfu` owns admission, LiveKit -room control, and token issue; `app` owns HTTP composition, site access, static -assets, and lifecycle; and `webassets` carries the embedded Browser bundle. +executes signaling and SFU effects under one lock; `sfu` owns admission and +embedded media through the shared Pion/LiveKit adapter; `app` owns HTTP +composition, site access, static assets, and lifecycle; and `webassets` carries +the embedded Browser bundle. `internal/app` keeps the native, launcher, and platform boundaries and supervises no server process. These are useful boundaries, not duplication to remove. @@ -39,7 +41,8 @@ requires a concrete ownership gain before further extraction. The graph controller is now universal. The former rollout switch and ordinary Host-star signaling path were removed together, so Hosted and Local -share one route authority and LiveKit remains an optional fallback of that graph. +share one route authority. Embedded SFU remains an optional fallback of that graph +under [ADR-0013](../adr/0013-embedded-node-local-media.md). ## Target Composition @@ -59,8 +62,8 @@ own their defaults, reachability and lifecycle policy. Each room keeps one serialized state owner. Go HTTP/WebSocket handlers must not mutate room, share and route maps independently. Run password derivation and -LiveKit I/O outside that owner, then apply results only while the exact session, -share and operation identities still match. Preserve the existing distinction +media-transport I/O outside that owner, then apply results only while the exact +session, share and operation identities still match. Preserve the existing distinction between decisions and effects rather than replacing Node serialization with unrelated locks around each map. @@ -70,10 +73,9 @@ unrelated locks around each map. [`embed`](https://pkg.go.dev/embed) serve HTTP and the built assets. - Signaling reuses the App's [`coder/websocket`](https://pkg.go.dev/github.com/coder/websocket). -- LiveKit room control and tokens use the standard library only: HS256 JWTs and - Twirp JSON requests. The official - [LiveKit Go SDK](https://github.com/livekit/server-sdk-go) adds a large module - and license surface for a handful of call shapes and is not used. +- Embedded SFU and Native media share `internal/media/forwarding`, which reuses + Pion transport and LiveKit media components. SDP/ICE uses the authenticated room + WebSocket; no external LiveKit room API or media-token service remains. - Room persistence keeps the current schema and transactions on the pure-Go [`modernc.org/sqlite`](https://pkg.go.dev/modernc.org/sqlite) driver, which builds without cgo and holds one exclusive connection; no ORM is needed. diff --git a/docs/todo.md b/docs/todo.md index 4957e481c..01a84d735 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -2,16 +2,46 @@ Last reviewed: 2026-10-07 -Only **Now** is executable. Product modules own behavior; Git/PRs own completed -history. A parked idea is not implementation authority. +The current 2.0 preparation includes independent investigation across this +ledger. Explicit owner holds remain in force; an investigation does not itself +accept a new feature or architecture. A missing device or reporter trace blocks +only that evidence claim, not unrelated work. Product modules own behavior; +Git/PRs own completed history. ## Now -- [ ] **2.0 candidate review and acceptance.** The owner reopened investigation, +- [ ] **2.0 candidate preparation and acceptance.** The owner reopened investigation, useful improvements and outstanding acceptance on 2026-10-07. Prepare one coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. +- [ ] **Optional interaction compatibility.** Apply the owner's chosen rule: + an unknown received reaction skips that visual effect while chat and sharing + continue. Current command/event decoding shares a strict reaction enum and a + decode error terminates signaling. Separate presentation support from malformed + or unauthorized traffic at the existing protocol owner; preserve bounded + payloads, membership/target validation and strict outgoing commands. Verify + the old/new reader boundary before adding another registered reaction. +- [ ] **Signaling execution and fairness.** Measure signaling-lock contention, + synchronous persistence, presence fan-out, diagnostic I/O and cross-room + password work before changing the lock or execution model. Start from the + existing bounded owners; do not introduce per-room actors without evidence. +- [ ] **Storage fault boundaries.** Extend real SQLite fault checks to disk I/O + failure during COMMIT/ROLLBACK. Write rejection, page-capacity exhaustion and + deferred-constraint COMMIT rejection are covered by the room tests. Verify + durable and in-memory room authority remain consistent and the next operation + has a defined outcome before choosing a recovery change; do not add + catch-and-continue retries. +- [ ] **Lifecycle ownership review.** Trace reachable per-share Host/Viewer, + source replacement, signaling recovery and media retirement paths under + [engineering review](./standards/engineering.md#ablation-and-review). + Reproduce C=3 structural-intent or multi-child evidence concerns before changing + their ownership. Keep Native bridge failure distinct from network failure; + account for preview, quality candidates, SFU updates and track-ended consumers + before changing Host loopback retirement. Simplify only for a concrete gain. +- [ ] **Documentation consistency.** Check the remaining guides, research and + owner links against current behavior; remove obsolete service/API descriptions + while retaining the scope and date of historical measurements. - [ ] **Native capture cadence acceptance.** The local Native SFU throughput rerun is limited by [low WGC arrival cadence](./research/native-client-media.md#virtual-display-acceptance-limit) reproduced on unchanged main and a separate Windows-API-only capture probe. @@ -38,26 +68,33 @@ history. A parked idea is not implementation authority. - [ ] **Windows ARM64 App.** Deferred by the owner on 2026-10-03. Keep Windows App packages and website downloads x64-only until this work is reopened. -## Deferred Feature Work +## Feature Assessments -These proposals remain outside the current candidate. Reassessment on -2026-10-07 did not establish enough benefit for their added owners and operating -cost. The media and transport assessments below remain recommendations for the -owner's final scope review, not permanent prohibitions. +Assess these proposals against the owner's benefit/complexity condition. +Existing negative results narrow the next experiment; they do not freeze the +whole candidate or require another version-number decision before other work. +Implementation needs a concrete useful result within the product model. - [ ] **4K and additional codecs.** The [media assessment](./research/native-client-media.md#media-capability-extension-assessment) - recommends retaining current limits for this candidate. Reopen with a concrete - source/receiver use case that justifies coordinated capture, forwarding and - resource bounds; keep UI presets separate from those bounds in any new design. - + recommends retaining current limits. Assess concrete source/receiver cases + before accepting coordinated capture, forwarding and resource-bound changes; + keep UI presets separate from those bounds in any new design. Additional + formats must account for late Viewers and relays that cannot decode or forward + the published format; codec negotiation alone does not transcode it. +- [ ] **Native Magicsock.** Go reuse is feasible, but the + [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) + have not established improved direct reachability. Seek a repeatable benefit + before accepting peer-key authorization/revocation, underlay readiness and + relay operation. Keep the isolated experiment separate from product routing; + the P2P evidence work below owns the network/interop acceptance boundary. - [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) identifies UDP/storage requirements but establishes no cost saving. Keep the existing P2P-only Demo until a provider and operating boundary are accepted. -- [ ] **Phone as a Host microphone: assess after the current work.** Explore an - opt-in link that pairs a phone's microphone with the Host's existing audio - mixer. Evaluate pairing/revocation, latency/echo, browser background limits +- [ ] **Phone as a Host microphone.** Assess an opt-in link that pairs a phone's + microphone with the Host's existing audio mixer. Evaluate pairing/revocation, + latency/echo, browser background limits and Browser/native input ownership before accepting an implementation. This - proposal does not reopen room voice or change the current release scope. + proposal does not reopen room voice; establish feasibility before accepting it. - [ ] **Passive App attachment: design hold.** Site mode authorizes one selected origin and supplies native media without starting a local room server. A passive replacement needs an accepted site-consent/discovery flow; it must not @@ -183,42 +220,27 @@ works, so a Rust alternative needs a concrete Go limitation. [Browser pool comparison](./research/browser-local-encoding-pool.md#balanced-startup-and-recovery) owns cold-encoder tradeoffs; low resolution alone is not a defect. Check whether a proposed move merely shifts pressure to another parent's siblings. -3. **Signaling execution model.** Measure signaling-lock contention, synchronous - persistence, presence fan-out, diagnostic I/O and cross-room password-work - fairness before changing the lock or execution model. -4. **Reachable ownership/refactor work.** With related behavior changes, evaluate - per-share Host and Viewer route ownership under - [engineering review](./standards/engineering.md#ablation-and-review). - Reopen C=3 structural-intent retention or multi-child evidence ownership only - with current-contract reproductions. Shared recovery must preserve Native - bridge versus network-failure semantics. Before changing Host loopback-failure - retirement, account for its preview, Browser quality candidates, quality/SFU - updates and track-ended consumers; independent Native publication alone does - not make ignoring the failure safe. This is not a diagnosed field cause. -5. **Storage fault recovery.** Choose and verify a damaged-disk/COMMIT/ROLLBACK - recovery policy before adding catch-and-continue or retries. This failure - boundary remains unestablished after ordinary persistence checks. -6. **Platform output.** Reopen for a registered receiver acting as an ordinary +3. **Platform output.** Reopen for a registered receiver acting as an ordinary Viewer only after the [platform-output gate](./research/platform-output.md) passes. -7. **Additional languages.** Review community catalogs and their rendered UI +4. **Additional languages.** Review community catalogs and their rendered UI following the [translation guide](./guide/translating.md), including names, menu navigation, text direction and layout. -8. **Windows code signing.** Revisit after enrollment in a trusted signing +5. **Windows code signing.** Revisit after enrollment in a trusted signing service. Sign executables before archive checksums; publisher identity does not guarantee that antivirus cloud scanning stops. -9. **Gitee download-source warning.** Paused by the owner. Keep GitHub primary +6. **Gitee download-source warning.** Paused by the owner. Keep GitHub primary and retain Gitee; do not add a self-hosted mirror. Chrome still blocks the Gitee attachment when Referer is removed. Reopen for new evidence or a provider review; the warning remains unresolved. -10. **Release-operation policy.** Protected release environments and immutable +7. **Release-operation policy.** Protected release environments and immutable draft assets remain unaccepted proposals; evaluate their benefit before adding release machinery. -11. **Automatic local chat history.** Explicit TXT export covers manual retention. +8. **Automatic local chat history.** Explicit TXT export covers manual retention. Reconsider automatic storage only with a stable room-incarnation identity, bounded retention and a clear delete control; reusable room codes must not combine conversations. No new history service or wire field is authorized. -12. **Direct OBS input (#436).** Reopen for a measured virtual-camera limitation. +9. **Direct OBS input (#436).** Reopen for a measured virtual-camera limitation. The [capture assessment](./research/camera-and-microphone.md) records raw-frame output and the WHIP boundary; virtual camera alone is not a second encode. Do not add an RTMP/WHIP listener without an accepted publication owner. diff --git a/internal/server/room/database_test.go b/internal/server/room/database_test.go index ac33dceab..3ed405fc1 100644 --- a/internal/server/room/database_test.go +++ b/internal/server/room/database_test.go @@ -4,12 +4,17 @@ import ( "bytes" "crypto/sha256" "database/sql" + "errors" + "fmt" "os" "path/filepath" "reflect" "strings" "testing" + "modernc.org/sqlite" + sqlite3 "modernc.org/sqlite/lib" + "github.com/TNTcraftHIM/Piik/internal/server/protocol" ) @@ -91,6 +96,121 @@ func TestDoesNotPersistARoomWhenPasswordDerivationIsBusy(t *testing.T) { } } +func TestRoomCreationStorageFailuresPreserveAuthority(t *testing.T) { + for _, testCase := range []struct { + name string + install string + remove string + code int + }{ + { + name: "read-only connection", install: "PRAGMA query_only = ON", + remove: "PRAGMA query_only = OFF", code: sqlite3.SQLITE_READONLY, + }, + { + // The room INSERT succeeds; a deferred constraint rejects COMMIT. + // Temporary objects leave the production schema unchanged. + name: "commit rejection", + install: `PRAGMA foreign_keys = ON; + CREATE TEMP TABLE fault_parent (id INTEGER PRIMARY KEY); + CREATE TEMP TABLE fault_child ( + id INTEGER REFERENCES fault_parent(id) DEFERRABLE INITIALLY DEFERRED + ); + CREATE TEMP TRIGGER reject_commit AFTER INSERT ON main.rooms + BEGIN INSERT INTO fault_child VALUES (1); END;`, + remove: "DROP TRIGGER temp.reject_commit", + code: sqlite3.SQLITE_CONSTRAINT_FOREIGNKEY, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + path := databasePath(t) + store := stableStore(t, path, Options{}) + createRoom(t, store, protocol.CodeEntryOpen, "", "4321") + before, err := store.database.readStoredRooms() + if err != nil { + t.Fatal(err) + } + free := len(store.freeRoomCodes) + if _, err := store.database.exec(testCase.install); err != nil { + t.Fatal(err) + } + _, err = tryCreateRoom(store, protocol.CodeEntryOpen, "", "5678") + var sqliteErr *sqlite.Error + if !errors.As(err, &sqliteErr) || sqliteErr.Code() != testCase.code { + t.Fatalf("create error = %v, want SQLite code %d", err, testCase.code) + } + after, err := store.database.readStoredRooms() + if err != nil || !reflect.DeepEqual(before, after) || store.Size() != 1 || len(store.freeRoomCodes) != free { + t.Fatalf("failed write changed authority: rooms=%d, free=%d, error=%v", store.Size(), len(store.freeRoomCodes), err) + } + if _, err := store.database.exec(testCase.remove); err != nil { + t.Fatal(err) + } + created := createRoom(t, store, protocol.CodeEntryOpen, "", "5678") + if created.RoomID != "5678" { + t.Fatalf("failed write did not release room code: %s", created.RoomID) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + restored := stableStore(t, path, Options{}) + if restored.Size() != 2 { + t.Fatalf("restored rooms = %d, want 2", restored.Size()) + } + if _, err := restored.HostManagedRoom(created.RoomID, created.HostToken); err != nil { + t.Fatalf("restored Host authority: %v", err) + } + }) + } +} + +func TestDatabaseFullDoesNotConsumeRoomAuthorityOrPreventLaterWrites(t *testing.T) { + path := databasePath(t) + store := stableStore(t, path, Options{MaxRooms: 1000}) + initialFree := len(store.freeRoomCodes) + pages, err := store.database.readPragmaInteger("page_count") + if err != nil { + t.Fatal(err) + } + if _, err := store.database.exec(fmt.Sprintf("PRAGMA max_page_count = %d", pages)); err != nil { + t.Fatal(err) + } + for n := 0; n < 1000; n++ { + roomID := fmt.Sprintf("%04d", 1000+n) + _, err := tryCreateRoom(store, protocol.CodeEntryOpen, "", roomID) + if err == nil { + continue + } + var sqliteErr *sqlite.Error + if !errors.As(err, &sqliteErr) || sqliteErr.Code() != sqlite3.SQLITE_FULL { + t.Fatalf("create error = %v, want SQLITE_FULL", err) + } + rows, err := store.database.readStoredRooms() + if err != nil || len(rows) != n || store.Size() != n || len(store.freeRoomCodes) != initialFree-n { + t.Fatalf("full database changed authority: rows=%d, rooms=%d, free=%d, error=%v", len(rows), store.Size(), len(store.freeRoomCodes), err) + } + if _, err := store.database.exec("PRAGMA max_page_count = 1000"); err != nil { + t.Fatal(err) + } + created := createRoom(t, store, protocol.CodeEntryOpen, "", roomID) + if created.RoomID != roomID { + t.Fatalf("recovered room code = %s, want %s", created.RoomID, roomID) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + restored := stableStore(t, path, Options{MaxRooms: 1000}) + if restored.Size() != n+1 { + t.Fatalf("restored rooms = %d, want %d", restored.Size(), n+1) + } + if _, err := restored.HostManagedRoom(created.RoomID, created.HostToken); err != nil { + t.Fatalf("restored Host authority: %v", err) + } + return + } + t.Fatal("bounded database did not reach SQLITE_FULL") +} + func TestRestoresTheExactAuthorityAggregateWithoutParticipants(t *testing.T) { path := databasePath(t) first := stableStore(t, path, Options{}) diff --git a/internal/server/signal/server_test.go b/internal/server/signal/server_test.go index 79db2d308..092fe005d 100644 --- a/internal/server/signal/server_test.go +++ b/internal/server/signal/server_test.go @@ -72,6 +72,71 @@ func TestCreateRoomReleasesLockWhenDerivationOrCommitPanics(t *testing.T) { } } +func TestPasswordPreparationDoesNotBlockAnotherRoom(t *testing.T) { + for _, update := range []bool{false, true} { + t.Run(fmt.Sprintf("update=%v", update), func(t *testing.T) { + started, release := make(chan struct{}), make(chan struct{}) + var hold atomic.Bool + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(unblock) + store, err := room.New(room.Options{ + MaxRooms: 3, MaxViewersPerRoom: 1, + Random: func(size int) []byte { + // Hold the existing salt source in the unlocked password phase. + if hold.CompareAndSwap(true, false) { + close(started) + <-release + } + return bytes.Repeat([]byte{byte(size)}, size) + }, + }) + if err != nil { + t.Fatal(err) + } + server := &Server{store: store} + original, err := server.CreateRoom(protocol.CodeEntryOpen, nil, "1234") + if err != nil { + t.Fatal(err) + } + hold.Store(true) + done := make(chan error, 1) + go func() { + password := "room-password" + var err error + if update { + _, err = server.UpdateRoomAccess(original.RoomID, original.HostToken, + protocol.SetViewerPasswordRequest{Password: &password}) + } else { + _, err = server.CreateRoom(protocol.CodeEntryPrivate, &password, "2345") + } + done <- err + }() + select { + case <-started: + case <-time.After(defaultWait): + t.Fatal("password preparation did not start") + } + if !server.mu.TryLock() { + t.Fatal("password work blocks signaling for other rooms") + } + server.mu.Unlock() + if _, err := server.CreateRoom(protocol.CodeEntryOpen, nil, "3456"); err != nil { + t.Fatalf("independent room creation: %v", err) + } + unblock() + select { + case err := <-done: + if err != nil { + t.Fatalf("password operation: %v", err) + } + case <-time.After(defaultWait): + t.Fatal("password operation did not finish") + } + }) + } +} + func TestStopSharingDoesNotWriteRoomAuthority(t *testing.T) { database, err := room.NewDatabase(filepath.Join(t.TempDir(), "rooms.sqlite")) if err != nil { From c2d32bae8df8ec468d201b34548610a368a0e9ba Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:34:43 +0800 Subject: [PATCH 015/262] fix(interactions): ignore unsupported reaction effects --- docs/research/room-interactions.md | 8 ++++++- docs/standards/rooms-access.md | 7 ++++++ docs/standards/versioning.md | 10 +++++--- docs/todo.md | 7 ------ internal/server/protocol/contract_test.go | 20 ++++++++++++++++ internal/server/protocol/interactions.go | 18 ++++++++++---- src/client/lib/room-interactions.ts | 22 ++++++++++------- src/shared/protocol.ts | 23 ++++++++++++------ src/shared/room-interactions.ts | 2 +- tests/fixtures/wire-samples.json | 10 ++++++++ tests/protocol.test.ts | 16 +++++++++++++ tests/signaling-admission.test.ts | 29 +++++++++++++++++++++++ 12 files changed, 141 insertions(+), 31 deletions(-) diff --git a/docs/research/room-interactions.md b/docs/research/room-interactions.md index a639269d8..82a56df1d 100644 --- a/docs/research/room-interactions.md +++ b/docs/research/room-interactions.md @@ -60,9 +60,15 @@ ends that service; merely leaving the Host page does not. session. - The viewer discovers this optional capability independently of first-picture startup. Late capability discovery enables data on the existing connection. -- A strict payload union admits plain chat text or a registered reaction. Chat is +- A strict command payload union admits plain chat text or a registered reaction. Chat is at most 280 Unicode code points, normalized to NFC, without control/bidi characters. It renders as text, with no markup, links, attachments or commands. +- Received reaction names are bounded lowercase ASCII identifiers, not arbitrary + HTML, assets or commands. The page ignores an unsupported effect after protocol + validation; malformed events still fail validation. This follows the command/ + response distinction in [Azure's extensible-enum guidance](https://github.com/microsoft/api-guidelines/blob/vNext/azure/Guidelines.md#enums--sdks-client-libraries), + checked on 2026-10-07. Existing strict pages still reject new names, so this + reader change alone does not authorize sending new reactions to those pages. - The server accepts at most one interaction per 800 ms per connection. It validates identity, subscription and target before fan-out. A slow recipient's data is skipped at 16 KiB queued output, reserving the existing bounded queue diff --git a/docs/standards/rooms-access.md b/docs/standards/rooms-access.md index d40a5b354..2431119fa 100644 --- a/docs/standards/rooms-access.md +++ b/docs/standards/rooms-access.md @@ -132,6 +132,13 @@ for participants still connected to the same room. Interaction subscriptions inherit existing membership, replacement and revocation checks; they grant no additional authority and do not modify media routes. +Sending a reaction requires a registered identifier. Received reaction events +validate their envelope, identity, target and bounded identifier, then the page +skips effects it cannot display. An unknown effect does not terminate signaling, +enter history, clear a draft or confirm a pending send. Unknown message kinds, +extra fields and malformed identities remain invalid; this presentation rule +does not make commands or access checks extensible. + Delivery is transient and bounded. The server keeps no message history; each page keeps only a bounded set of received messages and effects, without writing them to storage or diagnostics. Re-admission to the same room may retain that diff --git a/docs/standards/versioning.md b/docs/standards/versioning.md index ed20819d3..5b86801b5 100644 --- a/docs/standards/versioning.md +++ b/docs/standards/versioning.md @@ -63,8 +63,10 @@ parallel authority writers or blanket old-client fallback paths. ## Extending Interfaces -Command/event JSON objects still reject unknown keys and enums, and required -fields reject omissions. Therefore even adding a field can break the other side. +Command/event JSON objects still reject unknown keys, and required fields reject +omissions. Most enums remain closed; the bounded display-only reaction identifier +has an explicit [receive-side exception](./rooms-access.md#room-interactions). +Therefore even adding a field can break the other side. The former required `sfu` addition to strict `/api/capabilities` demonstrated both directions of this problem; an unchanged signaling identifier does not cover HTTP compatibility. @@ -75,7 +77,9 @@ capability means unavailable. Commands, credentials and authority checks remain strict. A new message type, enum value or behavior-changing field is emitted only after the other side advertises support; accepting unknown metadata is not permission to -execute unknown commands. Verify both readers before calling a change additive. +execute unknown commands. A more tolerant reader does not update already-released +readers: do not emit a new reaction to legacy pages merely because current pages +can skip it. Verify both readers before calling a change additive. Native discovery distinguishes an incompatible Piik App from an absent App and reports protocol numbers to local diagnostics; the source picker offers update/ diff --git a/docs/todo.md b/docs/todo.md index 01a84d735..4cd537ab1 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -15,13 +15,6 @@ Git/PRs own completed history. coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. -- [ ] **Optional interaction compatibility.** Apply the owner's chosen rule: - an unknown received reaction skips that visual effect while chat and sharing - continue. Current command/event decoding shares a strict reaction enum and a - decode error terminates signaling. Separate presentation support from malformed - or unauthorized traffic at the existing protocol owner; preserve bounded - payloads, membership/target validation and strict outgoing commands. Verify - the old/new reader boundary before adding another registered reaction. - [ ] **Signaling execution and fairness.** Measure signaling-lock contention, synchronous persistence, presence fan-out, diagnostic I/O and cross-room password work before changing the lock or execution model. Start from the diff --git a/internal/server/protocol/contract_test.go b/internal/server/protocol/contract_test.go index a3d74d33f..9e127ef29 100644 --- a/internal/server/protocol/contract_test.go +++ b/internal/server/protocol/contract_test.go @@ -159,6 +159,26 @@ func assertServerMessage(t *testing.T, name string, data []byte, want bool) { }) } +func TestFutureReactionEventsKeepPayloadAndAuthorityBounds(t *testing.T) { + for _, payload := range []string{ + `{"kind":"reaction","reaction":""}`, + `{"kind":"reaction","reaction":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}`, + `{"kind":"reaction","reaction":"future effect"}`, + `{"kind":"reaction","reaction":"👋"}`, + `{"kind":"reaction","reaction":"future-effect","targetPeerId":"bad"}`, + `{"kind":"reaction","reaction":"future-effect","text":"hidden command"}`, + `{"kind":"reaction","reaction":"tomato"}`, + `{"kind":"future-kind","reaction":"wave"}`, + } { + raw := `{"type":"room-interaction","id":"event_future_1234","requestId":"request_future_1234","occurredAt":1,` + + `"sender":{"peerId":"sender_peer_1234","role":"viewer","displayName":"Friend"},"payload":` + payload + `}` + assertServerMessage(t, payload, []byte(raw), false) + } + assertServerMessage(t, "future reaction with invalid sender", []byte( + `{"type":"room-interaction","id":"event_future_1234","requestId":"request_future_1234","occurredAt":1,`+ + `"sender":{"peerId":"bad","role":"viewer","displayName":"Friend"},"payload":{"kind":"reaction","reaction":"future-effect"}}`), false) +} + func assertClientMessage(t *testing.T, name string, data []byte, want bool) { t.Helper() t.Run(name, func(t *testing.T) { diff --git a/internal/server/protocol/interactions.go b/internal/server/protocol/interactions.go index 51f987ffe..737b78658 100644 --- a/internal/server/protocol/interactions.go +++ b/internal/server/protocol/interactions.go @@ -2,6 +2,7 @@ package protocol import ( "errors" + "regexp" "strings" "unicode/utf8" @@ -11,6 +12,8 @@ import ( const MaxChatCodePoints = 280 const InteractionIntervalMs = 800 +var reactionIDPattern = regexp.MustCompile(`^[a-z][a-z0-9-]{0,31}$`) + type InteractionPayload struct { Kind string `json:"kind"` Text string `json:"text,omitempty"` @@ -44,7 +47,8 @@ func (payload *InteractionPayload) UnmarshalJSON(data []byte) error { if err = fields.optional("targetPeerId"); err != nil { return err } - if fields.has("text") || (fields.has("targetPeerId") && !ValidOpaqueID(value.TargetPeerID)) { + if fields.has("text") || !reactionIDPattern.MatchString(value.Reaction) || + (fields.has("targetPeerId") && !ValidOpaqueID(value.TargetPeerID)) { return errors.New("invalid reaction payload") } switch value.Reaction { @@ -52,9 +56,6 @@ func (payload *InteractionPayload) UnmarshalJSON(data []byte) error { if !ValidOpaqueID(value.TargetPeerID) { return errors.New("throwing a prop requires a participant") } - case "wave", "heart", "clap", "laugh", "wow", "party", "fire", "eyes", "star", "sleep": - default: - return errors.New("unknown reaction") } default: return errors.New("unknown interaction kind") @@ -93,6 +94,15 @@ func decodeSendRoomInteraction(data []byte) (ClientMessage, error) { if !ValidOpaqueID(message.RequestID) { return nil, errors.New("invalid interaction requestId") } + // Only received presentation events are extensible. An unregistered command + // must never acquire meaning merely because its identifier is well-formed. + if message.Payload.Kind == "reaction" { + switch message.Payload.Reaction { + case "wave", "heart", "clap", "laugh", "wow", "party", "fire", "eyes", "star", "sleep", "tomato", "poop": + default: + return nil, errors.New("unknown reaction") + } + } return message, nil } diff --git a/src/client/lib/room-interactions.ts b/src/client/lib/room-interactions.ts index 073c41fb1..3f4d05397 100644 --- a/src/client/lib/room-interactions.ts +++ b/src/client/lib/room-interactions.ts @@ -3,7 +3,9 @@ import { INTERACTION_INTERVAL_MS, REACTION_DURATION_MS } from "../../shared/room import { createOpaqueId } from "./opaque-id"; import { identityHash } from "./identity-hash"; -export type RoomInteraction = Extract; +export type RoomInteraction = Omit, "payload"> & { + payload: InteractionPayload; +}; type Rejection = Extract["reason"]; export type InteractionError = Rejection | "offline" | "unconfirmed"; export const CHAT_OVERLAY_DURATION_MS = 6000; @@ -62,25 +64,29 @@ export class RoomInteractionSession { this.sendMessage({ type: "subscribe-room-interactions" }); } - receive(message: ServerMessage): boolean { + receive(event: ServerMessage): boolean { if (this.closed) return false; - if (message.type === "room-interactions-ready") { + if (event.type === "room-interactions-ready") { if (this.subscriptionStartedAt === null) return true; // One connection-scoped midpoint estimate; arrivals never move the clock. - this.serverOffset = message.serverTime - (this.subscriptionStartedAt + this.now()) / 2; + this.serverOffset = event.serverTime - (this.subscriptionStartedAt + this.now()) / 2; this.subscriptionStartedAt = null; this.update({ ready: true }); return true; } - if (message.type === "room-interaction-rejected") { - if (message.requestId === this.state.pending?.requestId) { + if (event.type === "room-interaction-rejected") { + if (event.requestId === this.state.pending?.requestId) { clearTimeout(this.pendingTimer); - this.update({ pending: null, error: message.reason }); + this.update({ pending: null, error: event.reason }); } return true; } - if (message.type !== "room-interaction") return false; + if (event.type !== "room-interaction") return false; if (!this.state.ready) return true; + const payload = interactionPayloadSchema.safeParse(event.payload); + // Unsupported decoration is not a signaling failure or a send confirmation. + if (!payload.success) return true; + const message: RoomInteraction = { ...event, payload: payload.data }; const now = this.now(); // Clock uncertainty may put a fresh event slightly ahead. Never extend its // lifetime beyond one full effect, and never replay an expired effect. diff --git a/src/shared/protocol.ts b/src/shared/protocol.ts index dbadef197..9a6b2cc7e 100644 --- a/src/shared/protocol.ts +++ b/src/shared/protocol.ts @@ -66,13 +66,22 @@ const opaqueIdSchema = z .max(128) .regex(/^[A-Za-z0-9_-]+$/); -export const interactionPayloadSchema = z.discriminatedUnion("kind", [ - z.object({ kind: z.literal("chat"), text: z.string().min(1).max(MAX_CHAT_CODE_POINTS * 2) - .refine(value => normalizeChatText(value) === value) }).strict(), - z.object({ kind: z.literal("reaction"), reaction: z.enum(REACTION_IDS), - targetPeerId: opaqueIdSchema.optional() }).strict(), -]).refine(payload => payload.kind !== "reaction" || !isThrow(payload.reaction) || !!payload.targetPeerId, +const chatPayloadSchema = z.object({ kind: z.literal("chat"), + text: z.string().min(1).max(MAX_CHAT_CODE_POINTS * 2) + .refine(value => normalizeChatText(value) === value) }).strict(); +const reactionPayloadSchema = z.object({ kind: z.literal("reaction"), + reaction: z.string().min(1).max(32).regex(/^[a-z][a-z0-9-]*$/), + targetPeerId: opaqueIdSchema.optional() }).strict() + .refine(payload => !isThrow(payload.reaction) || !!payload.targetPeerId, { message: "Throwing a prop requires a participant" }); +// Commands require a registered effect; received events may carry a future +// display-only identifier. Identity, payload shape and target bounds stay strict. +export const interactionPayloadSchema = z.discriminatedUnion("kind", [ + chatPayloadSchema, reactionPayloadSchema.safeExtend({ reaction: z.enum(REACTION_IDS) }), +]); +const receivedInteractionPayloadSchema = z.discriminatedUnion("kind", [ + chatPayloadSchema, reactionPayloadSchema, +]); export type InteractionPayload = z.infer; const noMediaRouteUpstreamSchema = z @@ -1051,7 +1060,7 @@ export const serverMessageSchema = z.union([ z.object({ type: z.literal("room-interaction"), id: opaqueIdSchema, requestId: opaqueIdSchema, occurredAt: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), sender: z.object({ peerId: opaqueIdSchema, role: z.enum(["host", "viewer"]), displayName: displayNameSchema }).strict(), - payload: interactionPayloadSchema }).strict(), + payload: receivedInteractionPayloadSchema }).strict(), authenticatedMessageSchema, z.object({ type: z.literal("sharing-start-failed"), diff --git a/src/shared/room-interactions.ts b/src/shared/room-interactions.ts index aa21f6d60..f0131fca6 100644 --- a/src/shared/room-interactions.ts +++ b/src/shared/room-interactions.ts @@ -4,7 +4,7 @@ export const INTERACTION_INTERVAL_MS = 800; export const REACTION_DURATION_MS = 2400; export const REACTION_IDS = ["wave", "heart", "clap", "laugh", "wow", "party", "fire", "eyes", "star", "sleep", "tomato", "poop"] as const; export type ReactionId = typeof REACTION_IDS[number]; -export const isThrow = (reaction: ReactionId) => reaction === "tomato" || reaction === "poop"; +export const isThrow = (reaction: string) => reaction === "tomato" || reaction === "poop"; export const REACTION_GLYPHS: Record = { wave: "👋", heart: "❤️", clap: "👏", laugh: "😂", wow: "🤯", party: "🎉", fire: "🔥", eyes: "👀", star: "⭐", sleep: "😴", tomato: "🍅", poop: "💩", diff --git a/tests/fixtures/wire-samples.json b/tests/fixtures/wire-samples.json index 12e6c6ee7..2d929e46d 100644 --- a/tests/fixtures/wire-samples.json +++ b/tests/fixtures/wire-samples.json @@ -40,6 +40,12 @@ "interactionIntervalMs": 800 }, "clientMessages": [ + { + "name": "send-unregistered-reaction", + "schema": "client", + "json": "{\"type\":\"send-room-interaction\",\"requestId\":\"request_12345678\",\"payload\":{\"kind\":\"reaction\",\"reaction\":\"future-effect\"}}", + "valid": false + }, { "name": "authenticate-host-minimal", "schema": "client", @@ -1050,6 +1056,10 @@ } ], "serverMessages": [ + { + "name": "room-interaction-future-reaction", + "json": "{\"type\":\"room-interaction\",\"id\":\"event_12345678\",\"requestId\":\"request_12345678\",\"occurredAt\":1700000000010,\"sender\":{\"peerId\":\"peer_12345678\",\"role\":\"host\",\"displayName\":\"Piik friend\"},\"payload\":{\"kind\":\"reaction\",\"reaction\":\"future-effect\",\"targetPeerId\":\"target_12345678\"}}" + }, { "name": "authenticated-host-minimal", "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v23\",\"peerId\":\"host_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"connectionId\":null,\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"none\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"role\":\"host\",\"viewerPasswordEnabled\":false}" diff --git a/tests/protocol.test.ts b/tests/protocol.test.ts index aeffaf411..28ce6b697 100644 --- a/tests/protocol.test.ts +++ b/tests/protocol.test.ts @@ -27,6 +27,22 @@ import { const token = "a".repeat(43); const roomId = "1234"; const viewerGrant = `${"b".repeat(21)}g`; + +it("accepts bounded future reaction events while keeping commands and known prop targets strict", () => { + const event = { type: "room-interaction", id: "event_future_1234", requestId: "request_future_1234", occurredAt: 1, + sender: { peerId: "sender_peer_1234", role: "viewer", displayName: "Friend" }, + payload: { kind: "reaction", reaction: "future-effect", targetPeerId: "target_peer_1234" } }; + expect(serverMessageSchema.safeParse(event).success).toBe(true); + expect(clientMessageSchema.safeParse({ type: "send-room-interaction", requestId: event.requestId, payload: event.payload }).success).toBe(false); + for (const payload of [ + { kind: "reaction", reaction: "" }, { kind: "reaction", reaction: "a".repeat(33) }, + { kind: "reaction", reaction: "future effect" }, { kind: "reaction", reaction: "👋" }, + { kind: "reaction", reaction: "future-effect", targetPeerId: "bad" }, + { kind: "reaction", reaction: "future-effect", text: "hidden command" }, + { kind: "reaction", reaction: "tomato" }, { kind: "future-kind", reaction: "wave" }, + ]) expect(serverMessageSchema.safeParse({ ...event, payload }).success).toBe(false); + expect(serverMessageSchema.safeParse({ ...event, sender: { ...event.sender, peerId: "bad" } }).success).toBe(false); +}); const qualitySettings = { resolution: "1080p", maxFramerate: 60, diff --git a/tests/signaling-admission.test.ts b/tests/signaling-admission.test.ts index f90dc9562..2fe5e8a60 100644 --- a/tests/signaling-admission.test.ts +++ b/tests/signaling-admission.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from "node:fs"; import { afterEach, describe, expect, it, vi } from "vitest"; import { SignalingClient } from "../src/client/lib/signaling"; import { DEFAULT_ROUTE_POLICY } from "../src/shared/protocol"; @@ -37,6 +38,34 @@ function fixture(role: "host" | "viewer" = "host") { } describe("room authentication and site admission", () => { + it("skips an unknown reaction without interrupting chat, media messages or a pending send", () => { + const { signal, sockets, events } = fixture(); + const interactions = signal.interactions!; + const wire = JSON.parse(readFileSync(new URL("./fixtures/wire-samples.json", import.meta.url), "utf8")); + const authenticated = JSON.parse(wire.serverMessages.find( + (sample: { name: string }) => sample.name === "authenticated-host-minimal").json); + sockets[0]!.receive(authenticated); + sockets[0]!.receive({ type: "room-interactions-ready", serverTime: Date.now() }); + expect(interactions.send({ kind: "chat", text: "keep my draft" })).toBe(true); + const before = interactions.getSnapshot(); + const event = { type: "room-interaction", id: "future_event_1234", requestId: before.pending!.requestId, + occurredAt: Date.now(), sender: { peerId: authenticated.peerId, role: "host", displayName: "Friend" }, + payload: { kind: "reaction", reaction: "future-effect" } }; + sockets[0]!.receive(event); + expect(interactions.getSnapshot()).toBe(before); + sockets[0]!.receive({ ...event, id: "chat_event_1234", payload: { kind: "chat", text: "keep my draft" } }); + expect(interactions.getSnapshot().messages).toHaveLength(1); + expect(interactions.getSnapshot().pending).toBeNull(); + const media = { type: "host-status", online: true, paused: false }; + sockets[0]!.receive(media); + expect(events.onMessage).toHaveBeenCalledWith(media); + expect(events.onTerminated).not.toHaveBeenCalled(); + expect(sockets[0]!.close).not.toHaveBeenCalled(); + expect(sockets).toHaveLength(1); + signal.stop(); + expect(vi.getTimerCount()).toBe(0); + }); + it("keeps one connection when starting the room during reconnect backoff", async () => { const { signal, sockets, events } = fixture(); sockets[0]!.dispatchEvent(Object.assign(new Event("close"), { code: 1006 })); From bbc176e51de1d4548909f515ea2004eb0d424817 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:48:15 +0800 Subject: [PATCH 016/262] refactor(storage): let SQLite own transaction state --- docs/research/cross-restart-room-recovery.md | 18 ++++++++ docs/todo.md | 6 --- internal/server/room/database.go | 48 ++++++-------------- 3 files changed, 32 insertions(+), 40 deletions(-) diff --git a/docs/research/cross-restart-room-recovery.md b/docs/research/cross-restart-room-recovery.md index 7a5c833d0..1d13677c9 100644 --- a/docs/research/cross-restart-room-recovery.md +++ b/docs/research/cross-restart-room-recovery.md @@ -101,6 +101,24 @@ compatibility path. The completed cutover is retained in Git; current 7. Surviving Host/Viewer tabs reauthenticate and rebuild a fresh route; no test labels that bounded rebuild as uninterrupted media. +### SQLite I/O Failure Boundary + +On 2026-10-07, the current pure-Go driver was exercised in an isolated Linux VM +with 600 synthetic rooms. Syscall fault injection rejected journal/database +`pwrite64` and `fsync` during commit and rollback, including spilled dirty pages +and sustained failures. Rejected writes did not become accepted authority; +while the fault persisted, further writes failed. After removing the fault, +reads, a new mutation and reopening the database retained the expected rows. +Read-only, page-capacity and deferred-constraint failures are also covered by +the ordinary room tests. This is not power-loss or filesystem-durability proof. + +SQLite can roll back automatically after an I/O or capacity error, so a separate +application transaction flag cannot reliably describe it. The owner now issues +one best-effort rollback on failure, preserves the original error and adds no +retry/reopen mechanism. This follows [SQLite's transaction error guidance](https://sqlite.org/lang_transaction.html#response_to_errors_within_a_transaction); +the experiment used [strace syscall fault injection](https://man7.org/linux/man-pages/man1/strace.1.html) +on disposable files, not production data. + ## Primary Sources - [Node.js SQLite](https://nodejs.org/docs/latest-v24.x/api/sqlite.html) diff --git a/docs/todo.md b/docs/todo.md index 4cd537ab1..5637ee027 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -19,12 +19,6 @@ Git/PRs own completed history. synchronous persistence, presence fan-out, diagnostic I/O and cross-room password work before changing the lock or execution model. Start from the existing bounded owners; do not introduce per-room actors without evidence. -- [ ] **Storage fault boundaries.** Extend real SQLite fault checks to disk I/O - failure during COMMIT/ROLLBACK. Write rejection, page-capacity exhaustion and - deferred-constraint COMMIT rejection are covered by the room tests. Verify - durable and in-memory room authority remain consistent and the next operation - has a defined outcome before choosing a recovery change; do not add - catch-and-continue retries. - [ ] **Lifecycle ownership review.** Trace reachable per-share Host/Viewer, source replacement, signaling recovery and media retirement paths under [engineering review](./standards/engineering.md#ablation-and-review). diff --git a/internal/server/room/database.go b/internal/server/room/database.go index 94657c0f7..199203f64 100644 --- a/internal/server/room/database.go +++ b/internal/server/room/database.go @@ -89,9 +89,6 @@ type Database struct { path string db *sql.DB conn *sql.Conn - // inTransaction reproduces DatabaseSync.isTransaction, which the driver - // does not expose; the ROLLBACK guard depends on it. - inTransaction bool } // NewDatabase is the RoomDatabase constructor. @@ -141,16 +138,16 @@ func (d *Database) Initialize() (restored []StoredRoomAuthority, err error) { if lockingMode != "exclusive" { return nil, errors.New("Room database could not acquire exclusive locking mode") } - if err = d.begin("BEGIN EXCLUSIVE"); err != nil { + if _, err = d.exec("BEGIN EXCLUSIVE"); err != nil { return nil, err } restored, err = d.recover() if err != nil { - d.rollbackIfNeeded() + d.rollback() return nil, err } - if err = d.finish("COMMIT"); err != nil { - d.rollbackIfNeeded() + if _, err = d.exec("COMMIT"); err != nil { + d.rollback() return nil, err } return restored, nil @@ -330,7 +327,7 @@ func (d *Database) Close() error { func (d *Database) closeHandles() error { conn, db := d.conn, d.db - d.conn, d.db, d.inTransaction = nil, nil, false + d.conn, d.db = nil, nil var connError error if conn != nil { connError = conn.Close() @@ -531,42 +528,25 @@ func (d *Database) transaction(operation func() error) error { if d.conn == nil { return errors.New("Room database is not initialized") } - if err := d.begin("BEGIN IMMEDIATE"); err != nil { + if _, err := d.exec("BEGIN IMMEDIATE"); err != nil { return err } if err := operation(); err != nil { - d.rollbackIfNeeded() + d.rollback() return err } - if err := d.finish("COMMIT"); err != nil { - d.rollbackIfNeeded() + if _, err := d.exec("COMMIT"); err != nil { + d.rollback() return err } return nil } -func (d *Database) begin(statement string) error { - if _, err := d.conn.ExecContext(context.Background(), statement); err != nil { - return err - } - d.inTransaction = true - return nil -} - -func (d *Database) finish(statement string) error { - if _, err := d.conn.ExecContext(context.Background(), statement); err != nil { - return err - } - d.inTransaction = false - return nil -} - -// rollbackIfNeeded is `if (database.isTransaction) database.exec("ROLLBACK")`: -// a ROLLBACK with no active transaction is itself an error. -func (d *Database) rollbackIfNeeded() { - if d.inTransaction { - _ = d.finish("ROLLBACK") - } +// SQLite may already have rolled back after an I/O or capacity error. An +// explicit rollback is harmless then; SQLite, not a Go mirror, owns that state. +// Preserve the original failure rather than replacing it with "no transaction". +func (d *Database) rollback() { + _, _ = d.exec("ROLLBACK") } func (d *Database) exec(query string, args ...any) (int64, error) { From b994c7b84de2140c0b90363128bdbe2b11e80993 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:59:27 +0800 Subject: [PATCH 017/262] docs(signal): record bounded-load execution evidence --- docs/research/server-consolidation.md | 25 +++++++++++++++++++++++++ docs/todo.md | 14 ++++++-------- internal/server/signal/server.go | 9 +++++---- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/docs/research/server-consolidation.md b/docs/research/server-consolidation.md index 167c1c9f0..9a76c68e1 100644 --- a/docs/research/server-consolidation.md +++ b/docs/research/server-consolidation.md @@ -101,6 +101,31 @@ and [ADR-0010](../adr/0010-cross-platform-client-runtime.md) as amended by ## Measured Before And After +### Signaling Execution Check + +On 2026-10-07, an isolated Windows loopback check used 10 rooms with one Host and +20 Viewers each. Nine Hosts changed their display names 10 times/s, exercising +presence encoding and fan-out; another task changed room policy 10 times/s, +and four password tasks ran once/s outside the signaling lock. The diagnostic +variant additionally wrote 200 synthetic route records/s through the real file +sink under that lock. Each run measured 200 requests from a separate room. + +| Storage / diagnostics | Signaling lock wait P95 | Request round-trip P95 | Retained connections | +| --- | ---: | ---: | ---: | +| Memory | Below clock resolution | 6.5 ms | 210/210 | +| SQLite | 1.0 ms | 5.9 ms | 210/210 | +| SQLite + diagnostic sink | 3.0 ms | 7.0 ms | 210/210 | + +These short local checks do not establish WAN, slow-disk or maximum-capacity +performance. A separate overload run at 900 name changes/s and 40 password +tasks/s reached 205 ms request P95 and lost 12 connections while bounded queues +were exercised; it is not a passing workload. The ordinary workload provides +no reason to introduce room actors or another mutation owner. Preserve the +bounded queues and unlocked password/media work; measure synchronous storage +and diagnostic sinks before changing execution for a larger deployment. + +### Original Package Consolidation + One machine, 2026-09-06: Windows 11, Node 24.15.0, Go 1.26.6. A script spawned each server with the same production configuration (`PORT=18787`, HTTPS public origin, site-access password, one STUN URL, loopback listen host), polled diff --git a/docs/todo.md b/docs/todo.md index 5637ee027..9fe87f199 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -15,10 +15,6 @@ Git/PRs own completed history. coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. -- [ ] **Signaling execution and fairness.** Measure signaling-lock contention, - synchronous persistence, presence fan-out, diagnostic I/O and cross-room - password work before changing the lock or execution model. Start from the - existing bounded owners; do not introduce per-room actors without evidence. - [ ] **Lifecycle ownership review.** Trace reachable per-share Host/Viewer, source replacement, signaling recovery and media retirement paths under [engineering review](./standards/engineering.md#ablation-and-review). @@ -29,6 +25,12 @@ Git/PRs own completed history. - [ ] **Documentation consistency.** Check the remaining guides, research and owner links against current behavior; remove obsolete service/API descriptions while retaining the scope and date of historical measurements. +- [ ] **Windows 11 capture border remains visible.** Reopened by a new report + on 2026-10-07. Recheck App/Browser capture ownership, border permission/API + results, live setting changes and concurrent captures before attributing this + to Windows. Obtain the affected version/path/build when available; the report + does not yet identify them. [Capture evidence](./research/native-client-lifecycle.md#windows-capture-borders) + owns the prior graceful-retirement and concurrent-capture checks. - [ ] **Native capture cadence acceptance.** The local Native SFU throughput rerun is limited by [low WGC arrival cadence](./research/native-client-media.md#virtual-display-acceptance-limit) reproduced on unchanged main and a separate Windows-API-only capture probe. @@ -146,10 +148,6 @@ of a reporter's cause; research owns the completed experiments and their limits. Separately retest the v1.5.0 App public-link report where one Viewer drops immediately after connection details appear while others work. Distinguish first-frame admission, signaling grace and current-edge recovery. -- [ ] **Windows 11 capture border remains visible.** Identify the App/Browser - capture path, Windows build, permission result and other active captures. - [Capture evidence](./research/native-client-lifecycle.md#windows-capture-borders) - owns the graceful-retirement and concurrent-capture checks. - [ ] **App discovery and share-start failures.** Retest missing sources and unreachable App reports across authorization, browser permissions, control capacity, enumeration and startup. For the v1.6.5 Auto-to-VP8 report, obtain diff --git a/internal/server/signal/server.go b/internal/server/signal/server.go index 38cdaba03..51d7b766f 100644 --- a/internal/server/signal/server.go +++ b/internal/server/signal/server.go @@ -6,10 +6,11 @@ // // Server.mu is the one lock of the effect layer: it guards every field of // the server and the router, the room.Store and the route controllers. Every -// handler, timer callback and goroutine takes it; it is released only around -// I/O (the password KDF and media transport), after which current session, +// handler, timer callback and goroutine takes it; password KDF and media +// transport work run unlocked, after which current session, // room and operation identities must be revalidated. Router methods and hooks are -// called with mu held and never lock. +// called with mu held and never lock. Room persistence and the optional route +// diagnostic sink are synchronous and belong in this lock's latency budget. package signal import ( @@ -45,7 +46,7 @@ const ( defaultMaxSignalConnections = 2_048 defaultMaxUnauthenticatedConnections = 256 // minSignalingChallengeIntervalMs is MIN_SIGNALING_CHALLENGE_INTERVAL_MS: - // the floor between two password challenges on one connection. + // the floor between two liveness challenges on one connection. minSignalingChallengeIntervalMs = int64(1_000) // serviceRestartCloseGrace is SERVICE_RESTART_CLOSE_GRACE_MS: how long a // peer has to acknowledge close{1012} before the socket is terminated. From 4854ffb2a55621decb58377d03d1fa0f59838ca3 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:04:01 +0800 Subject: [PATCH 018/262] fix(capture): retain border preference across permission changes --- docs/research/native-client-lifecycle.md | 11 +++++++++++ native/capture/windows/capture_border.h | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/research/native-client-lifecycle.md b/docs/research/native-client-lifecycle.md index 08303c358..4595b74e8 100644 --- a/docs/research/native-client-lifecycle.md +++ b/docs/research/native-client-lifecycle.md @@ -309,6 +309,17 @@ starts at build 20348, beyond consumer Windows 10 build 19045. The current capture helper therefore retains the system border there. Browser capture indicators remain Browser-owned. +The helper submits `IsBorderRequired(false)` after the access request completes, +including a denied result, as in the +[Windows capture sample](https://github.com/robmikh/Win32CaptureSample/blob/master/Win32CaptureSample/App.cpp). +Windows enforces permission and can honor the preference if consent changes +during capture. The earlier `Allowed` guard incorrectly made the first +permission result decide the session's lasting preference. The 2026-10-07 review +removed that guard; native checks and local capture/stop cycles with the border +preference off/on/off passed. A visible denied-to-allowed transition and the new +reporter's setup remain unverified; this correction does not establish the +reported cause or guarantee border visibility. + On Windows 11 build 26200, a 2026-09-21 controlled native-window check confirmed borderless permission was allowed and the default capture had no border. A second session explicitly requiring a border made it visible; gracefully ending diff --git a/native/capture/windows/capture_border.h b/native/capture/windows/capture_border.h index 433b30ff5..29e79627b 100644 --- a/native/capture/windows/capture_border.h +++ b/native/capture/windows/capture_border.h @@ -47,7 +47,9 @@ class CaptureBorder final { try { if (access_.Status() == winrt::Windows::Foundation::AsyncStatus::Started) return; const auto access = access_.GetResults(); - if (access == AccessStatus::Allowed) session.IsBorderRequired(false); + // Store the request even when consent is denied. Windows enforces current + // permission and can honor this preference if consent changes mid-share. + session.IsBorderRequired(false); std::osyncstream(std::cerr) << "capture-border-access=" << (access == AccessStatus::Allowed ? "allowed" : "not-allowed") << " status=" << static_cast(access) << '\n'; From 8bfffbd448b26689ca11b4207d0435155c2cb7af Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:10:11 +0800 Subject: [PATCH 019/262] test(route): verify bounded root convergence and failure ownership --- docs/standards/presentation-lifecycle.md | 8 ++ docs/todo.md | 7 -- .../server/route/root_convergence_test.go | 81 +++++++++++++++++++ 3 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 internal/server/route/root_convergence_test.go diff --git a/docs/standards/presentation-lifecycle.md b/docs/standards/presentation-lifecycle.md index 519136346..dc55a650c 100644 --- a/docs/standards/presentation-lifecycle.md +++ b/docs/standards/presentation-lifecycle.md @@ -241,6 +241,14 @@ use Browser reception; existing healthy peers are not torn down by this choice. Parent-network failure alone is not evidence that the local Native capability is unavailable. +The Native Host's local bridge remains required by its Browser share owner: +the stream supplies preview and Browser quality candidates and participates in +source and quality changes. A terminal failure retires that share through the +existing Host lifecycle; it is not merely a missing preview. An individual +downstream network failure instead recovers that edge and does not retire the +bridge or capture. Browser capture's optional Native fanout can fall back to +Browser senders because the Browser still owns its source. + ## Presence And Diagnostics Host and Viewer pages show the authoritative online Viewer count and roster, diff --git a/docs/todo.md b/docs/todo.md index 9fe87f199..52479718b 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -15,13 +15,6 @@ Git/PRs own completed history. coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. -- [ ] **Lifecycle ownership review.** Trace reachable per-share Host/Viewer, - source replacement, signaling recovery and media retirement paths under - [engineering review](./standards/engineering.md#ablation-and-review). - Reproduce C=3 structural-intent or multi-child evidence concerns before changing - their ownership. Keep Native bridge failure distinct from network failure; - account for preview, quality candidates, SFU updates and track-ended consumers - before changing Host loopback retirement. Simplify only for a concrete gain. - [ ] **Documentation consistency.** Check the remaining guides, research and owner links against current behavior; remove obsolete service/API descriptions while retaining the scope and date of historical measurements. diff --git a/internal/server/route/root_convergence_test.go b/internal/server/route/root_convergence_test.go new file mode 100644 index 000000000..bcbe5d5cd --- /dev/null +++ b/internal/server/route/root_convergence_test.go @@ -0,0 +1,81 @@ +package route + +import ( + "fmt" + "testing" +) + +func TestRootConvergenceRetainsCommittedRoutesUntilExactProof(t *testing.T) { + for _, capacity := range []int{2, 3} { + for childCount := 2; childCount <= capacity; childCount++ { + for _, outcome := range []string{"commit", "failure", "timeout", "session-replaced", "availability-preempts"} { + t.Run(fmt.Sprintf("capacity=%d/children=%d/%s", capacity, childCount, outcome), func(t *testing.T) { + routes := newController(capacity, Options{QualityConvergenceEnabled: true}) + addViewer(routes, B, capacity, nil) + routes.hydrateEdge(B, peerEdge(HOST, "b_from_host")) + children := []string{C, D, E}[:childCount] + for _, child := range children { + addViewer(routes, child, 0, nil) + routes.hydrateEdge(child, peerEdge(B, child+"_from_b")) + } + addViewer(routes, A, capacity, ms(10)) + routes.Reconcile(10) + commitCurrent(t, routes, 11, "a_from_host") + op := must(t, routes.Reconcile(13).Operation) + eq(t, op.Reason, DemandRootConvergence) + prepared := beginOperation(t, routes, BeginInput{ + NowMs: 14, ConnectionID: "root_candidate", Reservation: direct(), + }) + guard := CandidateGuard{ChildPeerID: op.ChildPeerID, ChildSessionID: op.ChildSessionID, + Revision: prepared.Current.Revision, ConnectionID: "root_candidate"} + + // Another child's report cannot authorize the prepared child, even + // when it names the candidate's revision and physical connection. + other := children[0] + if other == op.ChildPeerID { + other = children[1] + } + eq(t, routes.ObserveSenderQualityEvidence(senderSample(A, other, + prepared.Current.Revision, "root_candidate", "candidate-sender", + SenderQualityHealthy, 15), nil).Accepted, false) + ready := routes.CandidateReady(guard, 16, nil, CandidateProof{}) + eq(t, ready.Accepted, true) + eq(t, ready.Committed, false) + + switch outcome { + case "commit": + eq(t, routes.ObserveSenderQualityEvidence(senderSample(A, op.ChildPeerID, + prepared.Current.Revision, "root_candidate", "candidate-sender", + SenderQualityHealthy, 17), nil).Committed, true) + case "failure": + eq(t, routes.CandidateFailed(guard, 17).Accepted, true) + case "timeout": + eq(t, routes.OperationExpired(op.DeadlineAtMs).Accepted, true) + case "session-replaced": + routes.UpsertParticipant(viewerInput(A, "replacement_session", capacity), ms(17)) + eq(t, routes.CandidateReady(guard, 18, nil, CandidateProof{}).Accepted, false) + case "availability-preempts": + addViewer(routes, F, 0, ms(17)) + next := must(t, routes.Reconcile(18).Operation) + eq(t, next.ChildPeerID, F) + eq(t, next.Reason, DemandJoin) + eq(t, routes.CandidateReady(guard, 19, nil, CandidateProof{}).Accepted, false) + } + for _, child := range children { + edge := edgeOf(t, routes, child) + if child == op.ChildPeerID && outcome == "commit" { + eq(t, edge.ParentPeerID, A) + eq(t, edge.ConnectionID, "root_candidate") + } else { + eq(t, edge.ParentPeerID, B) + eq(t, edge.ConnectionID, child+"_from_b") + } + eq(t, edge.Usable, true) + eq(t, edge.PhysicalActive, true) + } + routes.assertGraph() + }) + } + } + } +} From 3204ca6f8af9072ef90f186a554c3c7a10be1f64 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:17:57 +0800 Subject: [PATCH 020/262] docs: align capture and deployment guidance with current owners --- cmd/piik-app/README.md | 4 ++-- cmd/piik-app/README.zh-CN.md | 2 +- docs/standards/configuration.md | 7 +++---- docs/status.md | 4 ++-- docs/todo.md | 3 --- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/cmd/piik-app/README.md b/cmd/piik-app/README.md index ad5bb8ec9..8b54e39e7 100644 --- a/cmd/piik-app/README.md +++ b/cmd/piik-app/README.md @@ -59,8 +59,8 @@ An App-opened Site remembers native activation, so later pages there may reuse the running App. Browser capture remains available without the App; see the [entry workflow](../../docs/standards/presentation-lifecycle.md#product-surface). -The Host page offers Browser capture and available native windows or screens; -select the source explicitly. Windows native capture offers VP8/Auto/H264; +The Host page offers Browser and Camera capture alongside available native +windows or screens; select the source explicitly. Windows native capture offers VP8/Auto/H264; Auto chooses one codec for the share. macOS and Linux native capture require hardware H264. Source and audio support vary by platform; their setup is covered in the [Windows](../../native/capture/windows/README.md), diff --git a/cmd/piik-app/README.zh-CN.md b/cmd/piik-app/README.zh-CN.md index a10268846..707768d50 100644 --- a/cmd/piik-app/README.zh-CN.md +++ b/cmd/piik-app/README.zh-CN.md @@ -47,7 +47,7 @@ App 会在启动时再次检查所选地址;它只影响本地邀请链接, 通过 App 打开的站点会记住原生功能的启用选择,之后打开该站点的页面可继续使用正在运行的 App。 没有 App 时仍可使用浏览器采集,详见[进入流程](../../docs/standards/presentation-lifecycle.md#product-surface)。 -房主页面提供浏览器采集及可用的原生窗口或屏幕,请明确选择来源。 +房主页面提供浏览器、摄像头采集,以及可用的原生窗口或屏幕,请明确选择来源。 Windows 原生采集提供 VP8/Auto/H264,Auto 会为本次分享选定一种编码。 macOS 和 Linux 原生采集需要硬件 H264。可选来源与音频支持取决于平台,设置要求见 [Windows](../../native/capture/windows/README.md)、[macOS](../../native/capture/darwin/README.md) diff --git a/docs/standards/configuration.md b/docs/standards/configuration.md index d9b9c32e6..e121012e0 100644 --- a/docs/standards/configuration.md +++ b/docs/standards/configuration.md @@ -275,12 +275,11 @@ When `NAT_PREDICTION_ENABLED=true`, the server derives `stun::3479` and `:3480` from the first ordinary STUN authority on UDP 3478. The Host sees a pre-share switch that defaults on and may disable it. The capability adds no media route or third-party service. It needs both -cloud security-group rules and the host's `/etc/nftables.conf` rule. Piik +cloud security-group rules and the host firewall. Piik binds every required UDP listener before opening room persistence or accepting signaling; a bind failure rolls back all newly owned sockets. Close and End -retire these listeners with the application. Existing coturn listeners must -be retired in the coordinated deployment because two processes cannot own the -same ports. Opening a cloud port without a listener has no effect, and local +retire these listeners with the application. No other service may own the same +listening ports. Opening a cloud port without a listener has no effect, and local binding alone does not prove external reachability. Disable the capability before removing firewall rules. Same-IP ports expose destination-port allocation behavior; a full RFC 5780 alternate-address diff --git a/docs/status.md b/docs/status.md index a2dab31e6..60690fee5 100644 --- a/docs/status.md +++ b/docs/status.md @@ -75,8 +75,8 @@ record own exact identity and postflight results. The existing deployment uses one Go process for Web, room authority/signaling, Binding-only STUN on UDP 3478/3479/3480 and optional SFU on UDP 7882. nginx owns -HTTPS; external LiveKit/coturn services remain disabled. Participants, routes and -media remain process-only. This private service is not a public demonstration. +HTTPS. Participants, routes and media remain process-only. This private service +is not a public demonstration. The separate US [public demo](https://demo.piik.tv) is deployed with open site entry, persistent room authority and P2P-only media. Cloudflare fronts HTTP/WSS diff --git a/docs/todo.md b/docs/todo.md index 52479718b..61915449d 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -15,9 +15,6 @@ Git/PRs own completed history. coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. -- [ ] **Documentation consistency.** Check the remaining guides, research and - owner links against current behavior; remove obsolete service/API descriptions - while retaining the scope and date of historical measurements. - [ ] **Windows 11 capture border remains visible.** Reopened by a new report on 2026-10-07. Recheck App/Browser capture ownership, border permission/API results, live setting changes and concurrent captures before attributing this From 8292f72a20cbeb6ecb6d07289d7112b823d02bde Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:40:00 +0800 Subject: [PATCH 021/262] docs(audio): assess phone microphone integration boundaries --- docs/research/camera-and-microphone.md | 22 ++++++++++++++++++++++ docs/todo.md | 10 +++++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/research/camera-and-microphone.md b/docs/research/camera-and-microphone.md index 9d5ea5225..b6d6eca69 100644 --- a/docs/research/camera-and-microphone.md +++ b/docs/research/camera-and-microphone.md @@ -124,6 +124,28 @@ ordinary invitation in another browser tab. Use headphones for same-device microphone tests. Browser-emulated mobile layout and synthetic devices do not establish actual iOS/Android permission, orientation, background or audio behavior. +## Phone Microphone Assessment + +The proposed phone link would supply one Host audio input, not give Viewers a +room-voice permission. Web Audio can mix a received `MediaStream` into the +Browser Host's existing output. The native mixer instead consumes fixed PCM +from local capture processes; it has no authenticated remote-input boundary. +Room signaling authorizes assigned parent/child connections, so an ordinary +Viewer invitation must not become permission to publish audio into the Host. + +A coherent implementation therefore needs a Host-scoped, revocable pairing +grant, one incoming audio connection and a decoded/resampled input for native +mixing. The existing Host share must own that input's mute, replacement and +retirement; input loss must leave healthy picture/source audio running. Phone +capture still requires a secure origin and explicit microphone permission. +The transport adds latency, and physical phone echo/background/lock behavior +is not established by Browser emulation or a loopback audio test. + +This is a source/API assessment, not an implemented or device-verified feature. +Keep it outside the current candidate until a concrete use case justifies that +pairing and input ownership. An audio device already exposed by the operating +system can use Piik's existing microphone selector without this extension. + ## Primary Technical References - [Google Meet audio controls](https://support.google.com/meet/answer/10409699?hl=en) diff --git a/docs/todo.md b/docs/todo.md index 61915449d..b1927466e 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -69,11 +69,11 @@ Implementation needs a concrete useful result within the product model. - [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) identifies UDP/storage requirements but establishes no cost saving. Keep the existing P2P-only Demo until a provider and operating boundary are accepted. -- [ ] **Phone as a Host microphone.** Assess an opt-in link that pairs a phone's - microphone with the Host's existing audio mixer. Evaluate pairing/revocation, - latency/echo, browser background limits - and Browser/native input ownership before accepting an implementation. This - proposal does not reopen room voice; establish feasibility before accepting it. +- [ ] **Phone as a Host microphone.** The [input assessment](./research/camera-and-microphone.md#phone-microphone-assessment) + identifies pairing/revocation and native decoded-input ownership beyond the + existing device selector. Keep it outside this candidate until a concrete + use case justifies that boundary and physical latency/echo/background checks. + This proposal does not reopen room voice. - [ ] **Passive App attachment: design hold.** Site mode authorizes one selected origin and supplies native media without starting a local room server. A passive replacement needs an accepted site-consent/discovery flow; it must not From e7cced717ab3574852d06ab8e14d5cc12e3a9979 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:49:01 +0800 Subject: [PATCH 022/262] docs(media): close physical-display throughput acceptance --- docs/research/native-client-media.md | 13 +++++++++++-- docs/todo.md | 5 ----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 274c441dd..003f617ad 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -379,8 +379,17 @@ Default, 4 ms and 1 ms capture intervals returned the same rate, including with the source kept topmost; property readback confirmed each requested interval. This localizes the limit to platform capture delivery in this environment but does not establish the virtual driver's cause or diagnose a field report. -Retain the gate's threshold and compare a physical display before changing capture scheduling, -encoder policy or route behavior. +No capture scheduling, encoder policy or gate threshold was changed. + +Later on the same day, Windows reported an active physical AOC display. The +unchanged Windows-API-only probe delivered about 32 fps, matching its GDI source, +at the default, 4 ms and 1 ms intervals. The candidate's Native H.264 two-room +SFU gate then passed the original 300-frame/30-second requirement, live profile +change, decoded audio, subscription recovery and retirement. Stopping the first +room preserved another 30 decoded frames and continuing audio in the second; +all owned processes, profiles and ports were released. This closes that local +throughput acceptance gap. It does not isolate the virtual driver's cause or +establish physical HDR, game-load or other-device performance. ## Native H.264 Motion Quality diff --git a/docs/todo.md b/docs/todo.md index b1927466e..adc4292d5 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -21,11 +21,6 @@ Git/PRs own completed history. to Windows. Obtain the affected version/path/build when available; the report does not yet identify them. [Capture evidence](./research/native-client-lifecycle.md#windows-capture-borders) owns the prior graceful-retirement and concurrent-capture checks. -- [ ] **Native capture cadence acceptance.** The local Native SFU throughput - rerun is limited by [low WGC arrival cadence](./research/native-client-media.md#virtual-display-acceptance-limit) - reproduced on unchanged main and a separate Windows-API-only capture probe. - Compare on a physical display before claiming current Native throughput - acceptance; do not lower the unchanged 300-frame/30-second requirement. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance are complete. Integrate through the next owner-approved publishing merge and verify the separate workflow with its original CI artifacts. Mirror failure From cea9eb21f0dcb9f2059fa37dc52344059584cc10 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:11:55 +0800 Subject: [PATCH 023/262] docs(server): describe current ownership and lifecycle invariants --- internal/server/room/database.go | 8 +- internal/server/room/password.go | 35 ++++----- internal/server/room/store.go | 109 ++++++++++++-------------- internal/server/route/api.go | 66 +++++++--------- internal/server/route/candidates.go | 7 +- internal/server/route/controller.go | 81 ++++++++----------- internal/server/route/graph.go | 7 +- internal/server/route/operation.go | 8 +- internal/server/route/quality.go | 4 +- internal/server/route/timing.go | 7 +- internal/server/signal/presence.go | 9 +-- internal/server/signal/router.go | 60 ++++++-------- internal/server/signal/router_pump.go | 7 +- internal/server/signal/server.go | 108 +++++++++---------------- internal/server/signal/session.go | 27 +++---- 15 files changed, 228 insertions(+), 315 deletions(-) diff --git a/internal/server/room/database.go b/internal/server/room/database.go index 199203f64..e57c04c15 100644 --- a/internal/server/room/database.go +++ b/internal/server/room/database.go @@ -170,7 +170,7 @@ func (d *Database) recover() ([]StoredRoomAuthority, error) { return d.readStoredRooms() } -// InsertRoom is insertRoom. +// InsertRoom persists one validated authority record atomically. func (d *Database) InsertRoom(room StoredRoomAuthority) error { if err := assertStoredRoom(room); err != nil { return err @@ -190,7 +190,7 @@ func (d *Database) InsertRoom(room StoredRoomAuthority) error { }) } -// SetViewerPassword is setViewerPassword; nil material clears the password. +// SetViewerPassword updates the exact Host's room; nil material clears the password. func (d *Database) SetViewerPassword( roomID string, hostTokenDigest, viewerPasswordMaterial []byte, ) error { @@ -213,7 +213,7 @@ func (d *Database) SetViewerPassword( }) } -// SetCodeEntryPolicy is setCodeEntryPolicy. +// SetCodeEntryPolicy persists code admission for the exact Host's room. func (d *Database) SetCodeEntryPolicy( roomID string, hostTokenDigest []byte, codeEntryPolicy protocol.CodeEntryPolicy, ) error { @@ -236,7 +236,7 @@ func (d *Database) SetCodeEntryPolicy( }) } -// SetViewerGrant is setViewerGrant. +// SetViewerGrant atomically replaces the grant digest and authorization generation. func (d *Database) SetViewerGrant( roomID string, hostTokenDigest, viewerGrantDigest []byte, viewerAuthorizationGeneration string, diff --git a/internal/server/room/password.go b/internal/server/room/password.go index 17841d264..5776a8d95 100644 --- a/internal/server/room/password.go +++ b/internal/server/room/password.go @@ -26,25 +26,21 @@ const ( gatePendingLimit = 16 ) -// errDerivationCancelled is the AsyncGate "cancelled" outcome. Callers never -// inspect it: they hand it to the commit method, which maps it the way the -// TypeScript call site did. +// errDerivationCancelled means the queued caller lost authority before its +// derivation started. The caller maps it to the operation's error. var errDerivationCancelled = errors.New("password derivation was cancelled") -// isGateOutcome reports whether err is one of the two AsyncGate outcomes rather -// than a thrown error. The TypeScript returned busy and cancelled as values, -// which its post-await checks inspected after re-reading the room; everything -// else threw out of the await. +// isGateOutcome identifies admission results that the commit boundary resolves +// after rechecking room authority. Other derivation errors return immediately. func isGateOutcome(err error) bool { return errors.Is(err, errDerivationCancelled) || hasCode(err, CodeRoomBusy) } -// DeriveViewerPasswordMaterial is createViewerPasswordMaterial: it validates the -// password, draws a fresh salt and returns salt||verifier. Call it WITHOUT the -// caller's lock held and hand both results to CreateRoom, ReplaceRoom or -// SetViewerPassword unchanged; those reproduce the TypeScript order in which a -// busy or cancelled outcome is turned into an error. mayStart may be nil, which -// is the TypeScript default of `() => true`. +// DeriveViewerPasswordMaterial validates the password, draws a fresh salt and +// returns salt||verifier. Call it without the caller's lock held and pass both +// results to CreateRoom, ReplaceRoom or SetViewerPassword unchanged so those +// commits can recheck authority before reporting busy/cancelled admission. +// A nil mayStart permits derivation once the gate admits it. func (s *Store) DeriveViewerPasswordMaterial( password string, mayStart func() bool, ) ([]byte, error) { @@ -62,10 +58,9 @@ func (s *Store) DeriveViewerPasswordMaterial( return slices.Concat(salt, verifier), nil } -// DeriveViewerPassword is the unlocked middle of connectViewerWithPassword: it -// derives the verifier for the salt ViewerPasswordChallenge returned. Unlike -// DeriveViewerPasswordMaterial it maps both gate outcomes itself, because the -// TypeScript threw them immediately after the await, before re-reading the room. +// DeriveViewerPassword is the unlocked password-admission step: it derives the +// verifier for the salt ViewerPasswordChallenge returned. Gate failures return +// before the room commit; cancelled admission is reported as an invalid token. func (s *Store) DeriveViewerPassword( password string, salt []byte, mayConnect func() bool, ) ([]byte, error) { @@ -89,7 +84,7 @@ func (s *Store) deriveViewerPassword( }) } -// gate is AsyncGate: at most limit derivations run at once and at most +// gate admits at most limit concurrent derivations and at most // pendingLimit wait, first in first out. A saturated gate answers ROOM_BUSY. type gate struct { mu sync.Mutex @@ -128,8 +123,8 @@ func (g *gate) acquire() bool { return true } -// release hands the slot to the first waiter instead of decrementing, which is -// what `const next = this.waiters.shift(); if (next) { next(); return; }` did. +// release transfers the active slot to the first waiter, keeping new callers +// behind already queued derivations. func (g *gate) release() { g.mu.Lock() if len(g.waiters) > 0 { diff --git a/internal/server/room/store.go b/internal/server/room/store.go index 34f79a73c..6c4dd5f30 100644 --- a/internal/server/room/store.go +++ b/internal/server/room/store.go @@ -18,15 +18,15 @@ import ( "github.com/TNTcraftHIM/Piik/internal/server/protocol" ) -// Capacity is ROOM_CAPACITY: every four-digit room code. +// Capacity is the number of four-digit room codes without a leading zero. const Capacity = 9_000 const roomCodeFirst = 1_000 -// ErrorCode is RoomStoreErrorCode. +// ErrorCode identifies a room authority or admission failure. type ErrorCode string -// The RoomStoreErrorCode set. +// Room authority and admission errors. const ( CodeInvalidToken ErrorCode = "INVALID_TOKEN" CodeRoomAccessDenied ErrorCode = "ROOM_ACCESS_DENIED" @@ -37,12 +37,12 @@ const ( CodeRoomLimit ErrorCode = "ROOM_LIMIT" ) -// Error is RoomStoreError. Its message is the code, as `super(code)` made it. +// Error carries a room authority or admission error code. type Error struct { Code ErrorCode } -// Error returns the code, which is the message `super(code)` produced. +// Error returns the code. func (e *Error) Error() string { return string(e.Code) } func codeError(code ErrorCode) error { return &Error{Code: code} } @@ -61,15 +61,14 @@ const ( type participant struct { clientID string peerID string - // sessionID is "" where the TypeScript left sessionId undefined. Session - // IDs are opaque IDs of at least 8 characters, so "" cannot collide. + // An empty sessionID means offline; validated wire IDs are never empty. sessionID string admittedBy string } // Room is one room's authority plus its participants. It has no exported // members: callers hold the pointer only to compare identity across the -// unlocked password derivation, as the TypeScript compared room objects. +// unlocked password derivation. type Room struct { roomID string hostTokenDigest []byte @@ -81,12 +80,12 @@ type Room struct { // Process-local: restored rooms receive a fresh reconnect window. Only the // last authenticated departure updates this; failed admission cannot renew it. emptySinceMs int64 - // ordering: JS Map insertion order decides viewer presence order, the + // Insertion order decides viewer presence order, the // closed-session list and the router's participant order. viewers ordered.Map[string, *participant] } -// CreatedRoom is CreatedRoom. ViewerGrant is "" where the TypeScript had null. +// CreatedRoom carries newly issued Host authority and an optional Viewer grant. type CreatedRoom struct { RoomID string HostToken string @@ -94,8 +93,8 @@ type CreatedRoom struct { ViewerGrant string } -// ConnectParticipantInput is ConnectParticipantInput; Token is read for a host -// and ViewerGrant for a viewer, "" meaning the key was absent. +// ConnectParticipantInput uses Token for a Host and ViewerGrant for a Viewer. +// Empty credentials are absent. type ConnectParticipantInput struct { RoomID string Role protocol.Role @@ -105,7 +104,7 @@ type ConnectParticipantInput struct { SessionID string } -// ConnectViewerWithPasswordInput is ConnectViewerWithPasswordInput. Password is +// ConnectViewerWithPasswordInput supplies Viewer password admission. Password is // consumed by the caller's DeriveViewerPassword call, not by the commit. type ConnectViewerWithPasswordInput struct { RoomID string @@ -114,8 +113,8 @@ type ConnectViewerWithPasswordInput struct { SessionID string } -// ConnectedParticipant is ConnectedParticipant. ReplacedSessionID is "" where -// the TypeScript omitted replacedSessionId. +// ConnectedParticipant records admission and any session it superseded. +// ReplacedSessionID is empty when no connected session was replaced. type ConnectedParticipant struct { RoomID string Role protocol.Role @@ -127,39 +126,35 @@ type ConnectedParticipant struct { ViewerAuthorizationGeneration string } -// DisconnectedParticipant is DisconnectedParticipant. type DisconnectedParticipant struct { RoomID string Role protocol.Role PeerID string } -// ConnectedPeer is ConnectedPeer. type ConnectedPeer struct { PeerID string SessionID string } -// ClosedRoom is ClosedRoom. type ClosedRoom struct { RoomID string SessionIDs []string } -// ReplacedRoom is ReplacedRoom. type ReplacedRoom struct { Created CreatedRoom Closed ClosedRoom } -// RevokedViewer is RevokedViewer; SessionID is "" where the TypeScript omitted -// sessionId. +// RevokedViewer identifies revoked admission. SessionID is empty for an offline Viewer. type RevokedViewer struct { PeerID string SessionID string } -// ViewerGrantUpdate is ViewerGrantUpdate. ViewerGrant is "" for a revoke. +// ViewerGrantUpdate carries a grant change and affected Viewers; ViewerGrant is +// empty when the invitation was revoked. type ViewerGrantUpdate struct { ViewerGrant string ViewerAuthorizationGeneration string @@ -167,13 +162,12 @@ type ViewerGrantUpdate struct { RevokedViewers []RevokedViewer } -// CodeEntryUpdate is CodeEntryUpdate. type CodeEntryUpdate struct { CodeEntryPolicy protocol.CodeEntryPolicy ViewerPasswordEnabled bool } -// Options is RoomStoreOptions. +// Options configures capacity, optional persistence and injectable entropy/time. type Options struct { MaxRooms int MaxViewersPerRoom int @@ -182,9 +176,9 @@ type Options struct { Now func() int64 } -// Store is RoomStore. See the package comment for the locking contract. +// Store owns room authority. See the package comment for the locking contract. type Store struct { - // ordering: JS Map insertion order decides AbandonAllRooms. + // Insertion order determines the order returned by AbandonAllRooms. rooms ordered.Map[string, *Room] freeRoomCodes []string random func(size int) []byte @@ -193,13 +187,11 @@ type Store struct { initialized bool maxRooms int maxViewersPerRoom int - // gate is per store; the TypeScript module-level gate was shared by every - // RoomStore in a process, of which there was one. + // The derivation gate bounds work across every room in this store. gate gate } -// New is the RoomStore constructor; the TypeScript threw where this returns an -// error. +// New validates limits and prepares a Store; Initialize loads persisted authority. func New(options Options) (*Store, error) { if options.MaxRooms <= 0 || options.MaxRooms > Capacity { return nil, fmt.Errorf("Room limit must be an integer between 1 and %d", Capacity) @@ -318,7 +310,7 @@ func (s *Store) BeginCreateRoom() error { return nil } -// CreateRoom is the commit half of createRoom. material and derived are the +// CreateRoom commits room creation. material and derived are the // results of DeriveViewerPasswordMaterial (both zero when no password was // given). func (s *Store) CreateRoom( @@ -327,7 +319,7 @@ func (s *Store) CreateRoom( derived error, preferredRoomID string, ) (CreatedRoom, error) { - // createViewerPasswordMaterial threw before any of the checks below. + // Derivation failures precede admission checks; gate outcomes are resolved below. if derived != nil && !isGateOutcome(derived) { return CreatedRoom{}, derived } @@ -359,8 +351,8 @@ func (s *Store) CreateRoom( return created, nil } -// HostManagedRoom is getHostManagedRoom: the room the exact Host token owns. -// It is the pre-KDF half of replaceRoom and setViewerPassword. +// HostManagedRoom returns the room owned by the exact Host token. The caller +// retains its identity while deriving password material outside the lock. func (s *Store) HostManagedRoom(roomID, hostToken string) (*Room, error) { if err := s.ensureInitialized(); err != nil { return nil, err @@ -368,7 +360,7 @@ func (s *Store) HostManagedRoom(roomID, hostToken string) (*Room, error) { return s.getHostManagedRoom(roomID, hostToken) } -// HostStillOwnsRoom is the mayStart closure replaceRoom and setViewerPassword +// HostStillOwnsRoom is the mayStart closure ReplaceRoom and SetViewerPassword // hand to the KDF gate: the room must still be the same object and the token // must still own it. Call it with the lock held. func (s *Store) HostStillOwnsRoom(roomID, hostToken string, current *Room) bool { @@ -376,7 +368,7 @@ func (s *Store) HostStillOwnsRoom(roomID, hostToken string, current *Room) bool return ok && room == current && verifyDigest(hostToken, room.hostTokenDigest) } -// ReplaceRoom is the commit half of replaceRoom. current is the room +// ReplaceRoom commits a replacement. current is the room // HostManagedRoom returned before the derivation. func (s *Store) ReplaceRoom( roomID, hostToken string, @@ -385,7 +377,7 @@ func (s *Store) ReplaceRoom( derived error, current *Room, ) (ReplacedRoom, error) { - // createViewerPasswordMaterial threw before the second getHostManagedRoom. + // Derivation failures return before rechecking authority; gate outcomes do not. if derived != nil && !isGateOutcome(derived) { return ReplacedRoom{}, derived } @@ -438,7 +430,7 @@ func (s *Store) replaceRoom( return ReplacedRoom{Created: created, Closed: closed}, nil } -// SetViewerPassword is the commit half of setViewerPassword; it reports whether +// SetViewerPassword commits a password change; it reports whether // a password is now set. current is the room HostManagedRoom returned before // the derivation. func (s *Store) SetViewerPassword( @@ -447,7 +439,7 @@ func (s *Store) SetViewerPassword( derived error, current *Room, ) (bool, error) { - // createViewerPasswordMaterial threw before the second getHostManagedRoom. + // Derivation failures return before rechecking authority; gate outcomes do not. if derived != nil && !isGateOutcome(derived) { return false, derived } @@ -474,7 +466,7 @@ func (s *Store) SetViewerPassword( return material != nil, nil } -// SetCodeEntryPolicy is setCodeEntryPolicy. +// SetCodeEntryPolicy updates code admission under exact Host authority. func (s *Store) SetCodeEntryPolicy( roomID string, policy protocol.CodeEntryPolicy, @@ -500,7 +492,7 @@ func (s *Store) SetCodeEntryPolicy( }, nil } -// SetViewerGrant is setViewerGrant; action is "rotate" or "revoke". +// SetViewerGrant rotates or revokes invitations and removes grant-admitted Viewers. func (s *Store) SetViewerGrant( roomID, action, hostToken string, ) (ViewerGrantUpdate, error) { @@ -550,7 +542,7 @@ func (s *Store) SetViewerGrant( room.viewerGrantDigest = viewerGrantDigest room.viewerAuthorizationGeneration = generation wasOccupied := roomOccupied(room) - // ordering: delete during iteration, which a JS Map allows. + // ordered.Map.All permits removal without skipping surviving entries. for clientID, viewer := range room.viewers.All() { if viewer.admittedBy == admittedByGrant { room.viewers.Delete(clientID) @@ -567,7 +559,7 @@ func (s *Store) SetViewerGrant( }, nil } -// ConnectParticipant is connectParticipant. +// ConnectParticipant authenticates a Host token, Viewer grant or open code entry. func (s *Store) ConnectParticipant(input ConnectParticipantInput) (ConnectedParticipant, error) { if err := s.ensureInitialized(); err != nil { return ConnectedParticipant{}, err @@ -599,7 +591,7 @@ func (s *Store) ConnectParticipant(input ConnectParticipantInput) (ConnectedPart return s.connectViewer(room, input.ClientID, input.SessionID, admittedByCode) } -// ViewerGrantMayEnter is viewerGrantMayEnter. +// ViewerGrantMayEnter validates an invitation without admitting a participant. func (s *Store) ViewerGrantMayEnter(roomID, grant string) (bool, error) { if err := s.ensureInitialized(); err != nil { return false, err @@ -615,12 +607,12 @@ func (s *Store) ViewerGrantMayEnter(roomID, grant string) (bool, error) { return viewerGrantIsValid(room, grant), nil } -// ViewerPasswordChallenge is the locked first half of connectViewerWithPassword: +// ViewerPasswordChallenge is the locked first half of password admission: // it returns the salt to derive against, a copy of the material to compare with // and the room reference to re-check afterwards. A missing room and a room // without a private password both answer with the same 48 zero -// bytes so the derivation cost does not reveal which. The only error is the -// `ensureInitialized()` the TypeScript ran before the password check. +// bytes so the derivation cost does not reveal which. An uninitialized store +// fails before this check. func (s *Store) ViewerPasswordChallenge( roomID string, ) (salt, expectedMaterial []byte, room *Room, err error) { @@ -639,7 +631,7 @@ func (s *Store) ViewerPasswordChallenge( return expectedMaterial[:viewerPasswordSaltBytes], expectedMaterial, found, nil } -// ConnectViewerWithPassword is the commit half of connectViewerWithPassword. +// ConnectViewerWithPassword commits password admission after derivation. // derived is the DeriveViewerPassword result; expectedMaterial and room come // from ViewerPasswordChallenge; mayConnect is evaluated with the lock held. func (s *Store) ConnectViewerWithPassword( @@ -678,8 +670,8 @@ func (s *Store) ConnectViewerWithPassword( return connected, err } -// DisconnectParticipant is disconnectParticipant; it returns nil where the -// TypeScript returned undefined. +// DisconnectParticipant clears only the matching session and returns nil when +// the room, peer or current session no longer matches. func (s *Store) DisconnectParticipant( roomID, peerID, sessionID string, ) (*DisconnectedParticipant, error) { @@ -715,7 +707,7 @@ func (s *Store) DisconnectParticipant( RoomID: roomID, Role: protocol.RoleViewer, PeerID: peerID}, nil } -// RemoveDisconnectedViewer is removeDisconnectedViewer. +// RemoveDisconnectedViewer removes a Viewer only if it has no current session. func (s *Store) RemoveDisconnectedViewer(roomID, peerID string) bool { room, ok := s.rooms.Get(roomID) if !ok { @@ -728,7 +720,7 @@ func (s *Store) RemoveDisconnectedViewer(roomID, peerID string) bool { return room.viewers.Delete(viewer.clientID) } -// GetConnectedHost is getConnectedHost. +// GetConnectedHost returns only a Host with a current session. func (s *Store) GetConnectedHost(roomID string) (ConnectedPeer, bool) { room, ok := s.rooms.Get(roomID) if !ok || room.host == nil || room.host.sessionID == "" { @@ -737,7 +729,7 @@ func (s *Store) GetConnectedHost(roomID string) (ConnectedPeer, bool) { return ConnectedPeer{PeerID: room.host.peerID, SessionID: room.host.sessionID}, true } -// GetConnectedViewer is getConnectedViewer. +// GetConnectedViewer returns only a Viewer with a current session. func (s *Store) GetConnectedViewer(roomID, peerID string) (ConnectedPeer, bool) { room, ok := s.rooms.Get(roomID) if !ok { @@ -779,8 +771,7 @@ func (s *Store) GetConnectedViewers(roomID string) []ConnectedPeer { return connected } -// AbandonRoom is abandonRoom; it returns nil where the TypeScript returned -// undefined. +// AbandonRoom retires room authority, returning nil if the room is already absent. func (s *Store) AbandonRoom(roomID string) (*ClosedRoom, error) { closed, err := s.abandonRooms([]string{roomID}) if err != nil || len(closed) == 0 { @@ -940,9 +931,8 @@ func (s *Store) takeRoomCode(preferredRoomID string) (string, error) { } // uniformIndex draws a uniform index below count from 8 random bytes, rejecting -// the values above the largest multiple of count. The TypeScript computed the -// limit in BigInt as 2^64 - 2^64 % count; (MaxUint64 % count + 1) % count is -// the same remainder without the uint64 wrap at a power-of-two count. +// the incomplete final bucket of the 2^64 possible values. +// (MaxUint64 % count + 1) % count computes 2^64 % count without overflowing. func uniformIndex(count int, random func(size int) []byte) (int, error) { span := uint64(count) remainder := (math.MaxUint64%span + 1) % span @@ -1063,8 +1053,7 @@ func (s *Store) ensureInitialized() error { return nil } -// writeDatabase runs a stable-authority write when a database is configured, -// which is the TypeScript `this.options.database?.write(...)`. +// writeDatabase runs a stable-authority write when a database is configured. func (s *Store) writeDatabase(write func() error) error { if s.database == nil { return nil diff --git a/internal/server/route/api.go b/internal/server/route/api.go index 9c41d2de6..1311e950d 100644 --- a/internal/server/route/api.go +++ b/internal/server/route/api.go @@ -6,9 +6,8 @@ import ( ) // Resource is the caller-owned SFU/overlap allocation the controller stores, -// moves between collections and compares by pointer identity (TS: the bare -// Resource type parameter, always used through JS Set semantics). The -// controller never reads its fields; they exist for the router and tests. +// moves between collections and compares by pointer identity. The controller +// never reads its fields; the router owns their meaning and release. type Resource struct { Kind ResourceKind // EndpointPeerID is set for ResourceOverlap. @@ -25,7 +24,7 @@ type Resource struct { Released bool } -// ResourceKind is the RouteResource discriminator of hybrid-media-router.ts. +// ResourceKind identifies the allocation owned by the router. type ResourceKind string // Resource kinds. @@ -36,7 +35,7 @@ const ( ) // UpstreamKind discriminates CandidateTuple, CommittedEdge, committedEdgeSeed -// and QualityUpstream (TS kind: "peer" | "sfu"). +// and QualityUpstream. type UpstreamKind string // Upstream kinds. @@ -45,7 +44,7 @@ const ( UpstreamSfu UpstreamKind = "sfu" ) -// Transport is the edge transport (TS transport: "direct" | "sfu"). +// Transport is the edge transport. type Transport string // Transports. @@ -54,7 +53,7 @@ const ( TransportSfu Transport = "sfu" ) -// PublicationPlan is the SFU tuple action (TS publication: "reuse" | "create" | "replace"). +// PublicationPlan is the SFU tuple action. type PublicationPlan string // Publication plans. @@ -93,8 +92,7 @@ const ( TransitionBoundedGap TransitionKind = "bounded-gap" ) -// SenderQualityState is the sender-side health sample state -// (TS "unknown" | "healthy" | "degraded"). +// SenderQualityState is the sender-side health sample state. type SenderQualityState string // Sender quality states. @@ -158,7 +156,7 @@ const ( // CandidateTuple is one route candidate: a direct edge from ParentPeerID // (Kind UpstreamPeer, Transport always TransportDirect, Regenerate = rebuild // the same edge with a fresh connection) or an SFU edge (Kind UpstreamSfu, -// Publication set). TS: CandidateTuple. +// Publication set). type CandidateTuple struct { Kind UpstreamKind ParentPeerID string @@ -171,7 +169,7 @@ type CandidateTuple struct { // BeginCurrentCandidate. Edge/Publication are set per Kind // (direct: neither; sfu-reuse: Edge, Borrowed when it is another holder's // live subscription and must not be released; sfu-create: Edge and -// Publication). Overlap is optional for every kind. TS: CandidateReservation. +// Publication). Overlap is optional for every kind. type CandidateReservation struct { Kind ReservationKind Edge *Resource @@ -183,7 +181,7 @@ type CandidateReservation struct { // EndpointRetirement is the producer retirement a bounded-gap transition // requires: an existing direct edge (RetireEdge: Child*/Parent*/Transport/ // ConnectionID) or the Host publication (RetirePublication: HostSessionID/ -// Generation/ConnectionID). TS: EndpointRetirement. +// Generation/ConnectionID). type EndpointRetirement struct { Kind RetirementKind ChildPeerID string @@ -198,15 +196,14 @@ type EndpointRetirement struct { // EndpointTransition says how the producer's copy budget absorbs the // candidate. ProducerPeerID is nil only for TransitionNone on an sfu/reuse -// plan (TS omitted it there and endpointTransitionEquals compares it). -// Retire is set only for TransitionBoundedGap. TS: EndpointTransition. +// plan. Retire is set only for TransitionBoundedGap. type EndpointTransition struct { Kind TransitionKind ProducerPeerID *string Retire *EndpointRetirement } -// CandidatePlan is a tuple with its endpoint transition. TS: CandidatePlan. +// CandidatePlan is a tuple with its endpoint transition. type CandidatePlan struct { Tuple CandidateTuple EndpointTransition EndpointTransition @@ -215,7 +212,7 @@ type CandidatePlan struct { // CommittedEdge is a child's one upstream edge in the committed graph. // Peer edges (Kind UpstreamPeer) set ParentPeerID/ParentSessionID and leave // Resource nil; SFU edges (Kind UpstreamSfu) set PublicationGeneration and -// Resource. TS: CommittedEdge. +// Resource. type CommittedEdge struct { Kind UpstreamKind ChildSessionID string @@ -229,7 +226,7 @@ type CommittedEdge struct { Resource *Resource } -// HostPublication is the single live SFU ingress. TS: HostPublication. +// HostPublication is the single live SFU ingress. type HostPublication struct { Generation string HostSessionID string @@ -246,7 +243,7 @@ type ConnectionAttemptProgress struct { Total int } -// CurrentAttempt is OperationSnapshot.current: the live candidate attempt. +// CurrentAttempt describes the operation's live candidate attempt. // ConnectionAttempt is nil when the attempt has no NAT opportunity. type CurrentAttempt struct { Tuple CandidateTuple @@ -256,7 +253,7 @@ type CurrentAttempt struct { } // OperationSnapshot is a deep copy of the in-flight operation plus the -// derived WakeAtMs. Current is nil without a live attempt. TS: OperationSnapshot. +// derived WakeAtMs. Current is nil without a live attempt. type OperationSnapshot struct { ChildPeerID string ChildSessionID string @@ -283,7 +280,7 @@ type RouteSnapshot struct { Operation *OperationSnapshot } -// ReconcileResult is the scheduler outcome. TS: ReconcileResult. +// ReconcileResult is the scheduler outcome. type ReconcileResult struct { Operation *OperationSnapshot RemovedPeerIDs []string @@ -292,8 +289,7 @@ type ReconcileResult struct { } // SettleResult is the outcome of candidate settlement or active replacement. -// Committed is true only when CandidateReady committed the attempt (TS: optional -// committed, read as truthy by the router). TS: SettleResult. +// Committed is true only when CandidateReady committed the attempt. type SettleResult struct { Accepted bool Committed bool @@ -302,9 +298,7 @@ type SettleResult struct { Released []*Resource } -// SenderQualityEvidenceResult is the outcome of the two sender-side -// observers. TS declares it separately with a required committed flag; the -// Go shape is identical to SettleResult. +// SenderQualityEvidenceResult is the settlement outcome of sender-side evidence. type SenderQualityEvidenceResult = SettleResult // BeginResult is the outcome of the cursor-guarded candidate calls. @@ -315,7 +309,7 @@ type BeginResult struct { Released []*Resource } -// ParticipantInput describes a connected participant. TS: ParticipantInput. +// ParticipantInput describes a connected participant. type ParticipantInput struct { PeerID string Role protocol.Role @@ -325,7 +319,7 @@ type ParticipantInput struct { // EdgeGuard identifies one committed edge exactly. ParentSessionID is only // compared for peer edges; callers pass the edge's own ParentSessionID for -// those and leave it empty for SFU edges. TS: EdgeGuard. +// those and leave it empty for SFU edges. type EdgeGuard struct { ChildPeerID string ChildSessionID string @@ -342,8 +336,7 @@ type ParentEdgeGuard struct { ConnectionID string } -// PublicationGuard identifies the Host publication exactly. TS: the inline -// guard of invalidateHostPublication and retireHostPublication. +// PublicationGuard identifies the Host publication exactly. type PublicationGuard struct { HostSessionID string RouteRevision int64 @@ -352,13 +345,13 @@ type PublicationGuard struct { } // AdoptDirectConnectionInput is an EdgeGuard plus the replacement -// connection. TS: EdgeGuard & { newConnectionId }. +// connection. type AdoptDirectConnectionInput struct { EdgeGuard NewConnectionID string } -// CandidateGuard identifies the live attempt exactly. TS: CandidateGuard. +// CandidateGuard identifies the live attempt exactly. type CandidateGuard struct { ChildPeerID string ChildSessionID string @@ -368,7 +361,7 @@ type CandidateGuard struct { // CandidateCursorGuard identifies one candidate slot of the current // operation: it must still be at Cursor with an equal Plan, built at -// FactVersion on BaseRevision. TS: CandidateCursorGuard. +// FactVersion on BaseRevision. type CandidateCursorGuard struct { ChildPeerID string ChildSessionID string @@ -380,8 +373,7 @@ type CandidateCursorGuard struct { // BeginInput is the BeginCurrentCandidate input. PublicationGeneration and // PublicationConnectionID are required for sfu create/replace tuples; -// HostSessionID is required for every SFU tuple (empty = TS undefined, -// rejected the same way). TS: the inline input of beginCurrentCandidate. +// HostSessionID is required for every SFU tuple. type BeginInput struct { Guard CandidateCursorGuard NowMs int64 @@ -398,7 +390,7 @@ type CandidateProof struct { } // QualityUpstream names the upstream a receiver-side window was measured -// on: PeerID is set only for UpstreamPeer. TS: RouteQualityEvidenceInput.upstream. +// on: PeerID is set only for UpstreamPeer. type QualityUpstream struct { Kind UpstreamKind PeerID string @@ -445,7 +437,7 @@ type SfuPublisherQualityEvidenceInput struct { } // Options configures a Controller. DebugRoomID empty disables the sanitised -// route debug events. TS: ControllerOptions. +// route debug events. type Options struct { HostPeerID string DebugRoomID string @@ -498,5 +490,5 @@ type Controller struct { retiringPublicationGenerations ordered.Map[string, struct{}] sfuBootstrapIntent *sfuBootstrapIntent consumedSfuBootstrapOpportunities ordered.Map[string, int] - rootConvergenceRootPeerID string // "" = TS undefined + rootConvergenceRootPeerID string } diff --git a/internal/server/route/candidates.go b/internal/server/route/candidates.go index 931e3d8d0..b6afa2ab8 100644 --- a/internal/server/route/candidates.go +++ b/internal/server/route/candidates.go @@ -353,8 +353,7 @@ func (c *Controller) descendantsOf(peerID string) []string { return descendants } -// withSelf ports `this.descendantsOf(peerId).add(peerId)`: the peer itself -// is appended last unless the walk already reached it. +// withSelf appends the peer itself unless the descendant walk already reached it. func withSelf(descendants []string, peerID string) []string { if slices.Contains(descendants, peerID) { return descendants @@ -508,8 +507,8 @@ func stringPtrEquals(left, right *string) bool { return *left == *right } -// In endpointTransitionEquals, `undefined !== HOST` matters for the -// none producer, hence the pointer comparison. +// endpointTransitionEquals distinguishes an absent producer from every peer, +// including the Host, even when no retirement is needed. func endpointTransitionEquals(left, right EndpointTransition) bool { if left.Kind != right.Kind { return false diff --git a/internal/server/route/controller.go b/internal/server/route/controller.go index 6463d7327..221b902be 100644 --- a/internal/server/route/controller.go +++ b/internal/server/route/controller.go @@ -20,18 +20,15 @@ const ( persistentDegradedWindows = int64(protocol.PersistentNativeEdgeDegradedWindows) ) -// candidateOpportunity is one consumed-ledger entry (TS CandidateOpportunity). +// candidateOpportunity records the attempts consumed at one candidate rank. type candidateOpportunity struct { rank int startedAttempts int exhausted bool } -// participant is a known peer (TS Participant). sessionID "" is the TS -// null: disconnected but in grace, or departed. Wire session IDs are -// validated opaque IDs, so an empty string never names a live session; -// bootstrapFailureReported and sfuFirstAtNextRoute are TS `true | undefined` -// and only ever compared to true, so a bool is exact. +// participant is a known peer. An empty sessionID means disconnected but in +// grace, or departed; validated wire IDs never name a live session with "". type participant struct { peerID string role protocol.Role @@ -45,13 +42,12 @@ type participant struct { sfuFirstAtNextRoute bool } -// sessionIs ports `participant.sessionId === id`: a disconnected -// participant (TS null) never equals any session string. +// sessionIs never matches a disconnected participant, including an empty id. func (p *participant) sessionIs(id string) bool { return p.sessionID != "" && p.sessionID == id } -// senderQualityIdentity is TS SenderQualityIdentity. +// senderQualityIdentity scopes quality evidence to one physical sender. type senderQualityIdentity struct { childSessionID string parentPeerID string @@ -60,10 +56,8 @@ type senderQualityIdentity struct { senderIdentity string } -// attempt is the live candidate (TS Attempt). The optional strings use "" -// for undefined (they are only ever set to non-empty values or compared -// after a truthiness check); the optional timestamps are pointers because -// 0 is a legal accepted/sample time and `!== undefined` is semantic. +// attempt is the live candidate. Optional identities use ""; optional +// timestamps use pointers because zero is a valid accepted/sample time. type attempt struct { tuple CandidateTuple revision int64 @@ -78,16 +72,16 @@ type attempt struct { transportConnected bool mediaReady bool connectionAttempt *ConnectionAttemptProgress - senderQualityState SenderQualityState // "" = undefined + senderQualityState SenderQualityState senderQualityAcceptedAtMs *int64 - senderQualityConsecutiveHealthyWindows int64 // read only through `?? 0` - senderQualityConsecutiveDegradedWindows int64 // read only through `?? 0` + senderQualityConsecutiveHealthyWindows int64 + senderQualityConsecutiveDegradedWindows int64 senderQualitySampleTimestampMs *int64 - senderQualityIdentity string // "" = undefined + senderQualityIdentity string relativeQualityApprovedAtMs *int64 } -// operation is the single in-flight route operation (TS ChildOperation). +// operation is the single in-flight route operation. type operation struct { childPeerID string childSessionID string @@ -105,9 +99,8 @@ type operation struct { current *attempt } -// planAt ports `operation.candidates[index]`: nil when out of range. It -// returns a copy because the candidate array is spliced in place and TS -// plan objects are never mutated after construction. +// planAt returns nil when out of range and otherwise copies the plan so callers +// do not retain a pointer into the candidate slice while it is reordered. func (o *operation) planAt(index int) *CandidatePlan { if index < 0 || index >= len(o.candidates) { return nil @@ -116,13 +109,13 @@ func (o *operation) planAt(index int) *CandidatePlan { return &plan } -// planAtOrLast ports `operation.candidates[Math.min(cursor, length - 1)]`, -// which is undefined (nil) for an empty array. +// planAtOrLast returns the cursor's plan, the final plan if exhausted, or nil +// when there are no candidates. func (o *operation) planAtOrLast() *CandidatePlan { return o.planAt(min(o.cursor, len(o.candidates)-1)) } -// currentTuple ports `operation.current?.tuple ?? operation.candidates[operation.cursor]?.tuple`. +// currentTuple prefers the live attempt over the next planned candidate. func (o *operation) currentTuple() *CandidateTuple { if o.current != nil { tuple := o.current.tuple @@ -134,8 +127,7 @@ func (o *operation) currentTuple() *CandidateTuple { return nil } -// directContinuation is TS DirectContinuation; parentPeerIDs is the retry -// order and is spliced in place. +// directContinuation retains the remaining parent retry order behind an SFU edge. type directContinuation struct { childSessionID string sfuConnectionID string @@ -154,8 +146,8 @@ type sfuBootstrapCarrier struct { carrierPeerID string } -// routeTimingRecord is TS RouteTimingRecord; the optional timestamps are -// pointers (undefined = never happened, distinct from 0). +// routeTimingRecord uses nil timestamps for events that have not happened; +// zero is a valid event time. type routeTimingRecord struct { demandAtMs int64 reason DemandReason @@ -167,8 +159,8 @@ type routeTimingRecord struct { rejectionBucket RejectionBucket } -// routeQualityObservation is TS RouteQualityObservation. upstreamPeerID is -// "" for the sfu upstream (TS null); lastDecodedProgressAtMs nil is TS null. +// routeQualityObservation belongs to one receiver edge and presentation epoch. +// upstreamPeerID is empty for SFU; nil lastDecodedProgressAtMs means no progress. type routeQualityObservation struct { childSessionID string upstreamKind UpstreamKind @@ -186,8 +178,8 @@ type routeQualityObservation struct { lastDecodedProgressAtMs *int64 } -// senderQualityObservation is TS SenderQualityObservation. The consumed -// ledger is a pointer because `{...previous}` copies carry the same Map. +// senderQualityObservation copies share the consumed-candidate ledger, so a new +// sample on the same physical sender cannot reopen an exhausted opportunity. type senderQualityObservation struct { childSessionID string parentPeerID string @@ -210,7 +202,7 @@ type sfuPublisherQualityObservation struct { lastAcceptedAtMs int64 } -// resourceSet is a JS Set: pointer identity plus insertion order. +// resourceSet deduplicates by pointer identity and preserves insertion order. type resourceSet struct { seen map[*Resource]struct{} list []*Resource @@ -234,12 +226,12 @@ func (s *resourceSet) has(resource *Resource) bool { return ok } -// resources returns a fresh slice in insertion order (`[...set]`). +// resources returns a fresh slice in insertion order. func (s *resourceSet) resources() []*Resource { return append([]*Resource{}, s.list...) } -// concatResources ports `[...a, ...b]` into a fresh, never-nil slice. +// concatResources concatenates into a fresh, never-nil slice. func concatResources(parts ...[]*Resource) []*Resource { out := []*Resource{} for _, part := range parts { @@ -248,7 +240,7 @@ func concatResources(parts ...[]*Resource) []*Resource { return out } -// dedupeStrings ports `[...new Set(values)]`: first occurrence wins. +// dedupeStrings retains each value's first occurrence. func dedupeStrings(values []string) []string { seen := make(map[string]struct{}, len(values)) out := make([]string, 0, len(values)) @@ -262,8 +254,7 @@ func dedupeStrings(values []string) []string { return out } -// New ports the constructor: panics with "Endpoint media copy capacity must -// be 1, 2, or 3" or "Route operation timeout must be a positive integer". +// New constructs a controller and panics on invalid capacity, timeout or SFU policy. func New(options Options) *Controller { if err := protocol.AssertEndpointMediaCopyCapacity(options.EndpointMediaCopyCapacity); err != nil { panic(err) @@ -476,7 +467,7 @@ func (c *Controller) UpsertParticipant(input ParticipantInput, nowMs *int64) []* } return released } - // A new key appends to the insertion order (TS Map.set at 573). + // New participants append to the scheduling order. c.participants.Set(input.PeerID, &participant{ peerID: input.PeerID, role: input.Role, @@ -899,15 +890,13 @@ func (c *Controller) allocateRevision() int64 { return c.latestRevision } -// isSafeInteger is Number.isSafeInteger for a value that is already integral. +// isSafeInteger keeps values exactly representable by the Browser's number type. func isSafeInteger(value int64) bool { return value >= -maxSafeInteger && value <= maxSafeInteger } -// joinOrder is allocated once per participant and preserved across -// rebindHostIdentity, so two live -// participants never share one and the TS localeCompare (ICU) tiebreak is -// unreachable; strings.Compare stands in for it. +// compareParticipant preserves join order, including across Host rebinding. +// Live participants have unique orders; peer ID breaks ties in synthetic inputs. func compareParticipant(left, right *participant) int { if left.joinOrder != right.joinOrder { if left.joinOrder < right.joinOrder { @@ -929,9 +918,7 @@ func stablePairRank(childPeerID, parentPeerID string) int64 { return rank } -// JS Math.round is half-up while math.Round is half-away-from-zero; -// they differ only for negative halves, which the -// `rounded < 0` branch drops either way. NaN and +/-Inf drop the window. +// safeAdd ignores invalid rounded deltas and saturates at the wire integer bound. func safeAdd(total int64, delta float64) int64 { rounded := math.Round(delta) if math.IsNaN(rounded) || math.IsInf(rounded, 0) || rounded < 0 || rounded > float64(maxSafeInteger) { diff --git a/internal/server/route/graph.go b/internal/server/route/graph.go index 124c01933..97b9aa501 100644 --- a/internal/server/route/graph.go +++ b/internal/server/route/graph.go @@ -53,7 +53,7 @@ func (c *Controller) pruneDepartedLeaves(released *[]*Resource) []string { changed := true for changed { changed = false - // Live iteration: deleting the visited entry is JS-safe (ordered.Map.All). + // ordered.Map.All permits deleting the visited entry during iteration. for _, current := range c.participants.All() { if current.role != protocol.RoleViewer || !current.departureConfirmed || len(c.childrenOf(current.peerID)) > 0 { continue @@ -167,8 +167,7 @@ func (c *Controller) removePublicationGeneration(generation string) []*Resource return released } -// retainsAnchor is the TS default argument of retireSfuEdge -// (`this.childrenOf(viewerPeerId).length > 0`), evaluated at call time. +// retainsAnchor keeps a retiring SFU edge while it still has peer children. func (c *Controller) retainsAnchor(viewerPeerID string) bool { return len(c.childrenOf(viewerPeerID)) > 0 } @@ -191,7 +190,7 @@ func (c *Controller) retireSfuEdge(viewerPeerID string, edge *CommittedEdge, rel } // hasRetiringAnchor reports whether an inactive SFU edge still references -// the generation (the `some` at 4816 and 4880). +// the generation. func (c *Controller) hasRetiringAnchor(generation string) bool { for _, edge := range c.upstreamByViewer.Values() { if edge.Kind == UpstreamSfu && !edge.PhysicalActive && edge.PublicationGeneration == generation { diff --git a/internal/server/route/operation.go b/internal/server/route/operation.go index 00588ec76..10bc58fb0 100644 --- a/internal/server/route/operation.go +++ b/internal/server/route/operation.go @@ -9,7 +9,7 @@ import ( // validation is the private validateOrAdvance result. type validation struct { released []*Resource - exhaustedChildPeerID string // "" = undefined + exhaustedChildPeerID string expired bool consumedGuard bool } @@ -353,7 +353,7 @@ func (c *Controller) NoteCurrentCandidateRejection(guard CandidateCursorGuard, b } // SkipCurrentCandidate consumes the guarded candidate and advances the -// cursor (TS defaulted bucket to "stale"; Go callers pass RejectionStale). +// cursor, recording the caller's rejection reason. func (c *Controller) SkipCurrentCandidate(guard CandidateCursorGuard, nowMs int64, bucket RejectionBucket) BeginResult { validation := c.validateOrAdvance(nowMs, nil) op := c.operation @@ -1163,7 +1163,7 @@ func (c *Controller) replanRemaining(op *operation, firstTuple, restoreTuple *Ca } // operationSnapshot returns a deep copy plus the derived wakeAtMs; -// Current omits ConnectionAttempt when undefined. +// Current omits ConnectionAttempt when the attempt has no NAT opportunity. func (c *Controller) operationSnapshot() *OperationSnapshot { op := c.operation if op == nil { @@ -1440,7 +1440,7 @@ func (c *Controller) consumeDirectContinuationCandidate(op *operation) { if len(continuation.parentPeerIDs) == 0 { c.directContinuations.Delete(op.childPeerID) } else { - // TS delete + set: the entry moves to the tail (round-robin state). + // Give the next continuation a turn before retrying this child. c.directContinuations.MoveToBack(op.childPeerID, continuation) } } diff --git a/internal/server/route/quality.go b/internal/server/route/quality.go index 5ac7dacbf..e8c9fce42 100644 --- a/internal/server/route/quality.go +++ b/internal/server/route/quality.go @@ -386,7 +386,7 @@ func (c *Controller) ObserveSfuPublisherQualityEvidence(input SfuPublisherQualit } // settleCandidateQuality folds a sender sample into the live -// candidate and fails or commits it. senderIdentity "" is the TS null. +// candidate and fails or commits it. An empty senderIdentity means unknown. func (c *Controller) settleCandidateQuality(op *operation, att *attempt, state SenderQualityState, acceptedAtMs int64, senderIdentity string, sampleTimestampMs *int64, commitReservation func(CandidateReservation) bool) SenderQualityEvidenceResult { if att.senderQualityState != state { c.debug("candidate-quality-observed", @@ -796,7 +796,7 @@ func qualityUpstreamMatches(edge *CommittedEdge, upstream QualityUpstream) bool return upstream.Kind == UpstreamSfu } -// qualityObservationMatchesEdge treats an empty upstreamPeerID as TypeScript null. +// qualityObservationMatchesEdge requires an empty upstreamPeerID for SFU. func qualityObservationMatchesEdge(observation *routeQualityObservation, edge *CommittedEdge) bool { if edge.Kind == UpstreamPeer { return observation.upstreamKind == UpstreamPeer && observation.upstreamPeerID == edge.ParentPeerID diff --git a/internal/server/route/timing.go b/internal/server/route/timing.go index 495bf7230..28d09f677 100644 --- a/internal/server/route/timing.go +++ b/internal/server/route/timing.go @@ -140,7 +140,7 @@ func (c *Controller) diagnosticParent(edge *CommittedEdge, ordinals map[string]i return protocol.RouteDiagnosticParent{Kind: "viewer", Ordinal: protocol.Int(ordinal)} } -// diagnosticQuality returns nil (TS null) unless the observation is +// diagnosticQuality returns nil unless the observation is // fresh, non-empty and still describes the current edge. func (c *Controller) diagnosticQuality(childPeerID string, edge *CommittedEdge, nowMs int64) *protocol.RouteDiagnosticQuality { observation := c.qualityObservations[childPeerID] @@ -170,9 +170,8 @@ func (c *Controller) diagnosticQuality(childPeerID string, edge *CommittedEdge, } } -// elapsedMs returns nil (TS null) when the end precedes the start or -// the difference is not a safe integer; the inputs are integral so -// Math.floor is the identity. +// elapsedMs returns nil when the end precedes the start or the difference +// exceeds the wire integer bound. func elapsedMs(startedAtMs, endedAtMs int64) *protocol.Int { elapsed := endedAtMs - startedAtMs if !isSafeInteger(elapsed) || elapsed < 0 { diff --git a/internal/server/signal/presence.go b/internal/server/signal/presence.go index 5859ba6ef..94f484a58 100644 --- a/internal/server/signal/presence.go +++ b/internal/server/signal/presence.go @@ -14,9 +14,7 @@ import ( "github.com/TNTcraftHIM/Piik/internal/server/room" ) -// viewerKey is the TS `${roomId}:${peerId}` used by the grace timers and the -// viewer quality evidence maps (viewerGraceKey and viewerConnectionKey were -// the same expression). +// viewerKey scopes grace timers and quality evidence to one room participant. func viewerKey(roomID, peerID string) string { return roomID + ":" + peerID } // sendViewerPresence lists the host entry first @@ -158,9 +156,8 @@ func (s *Server) handleDisconnect(sess *session) { s.sendViewerPresence(disconnected.RoomID) - // The viewer grace timer. The registered-pointer check stands in - // for clearTimeout; the store's removeDisconnectedViewer is the identity - // guard against a viewer that reconnected meanwhile. + // The registered pointer rejects stale grace callbacks; the store then + // guards against removing a Viewer that has already reconnected. roomID, peerID := disconnected.RoomID, disconnected.PeerID key := viewerKey(roomID, peerID) timer := &graceTimer{} diff --git a/internal/server/signal/router.go b/internal/server/signal/router.go index f6cda6bba..701a4ebed 100644 --- a/internal/server/signal/router.go +++ b/internal/server/signal/router.go @@ -1,7 +1,7 @@ package signal // The router owns no lock. signal.Server.mu (routerOptions.mu) is held by the -// caller of every method except close, hooks are called with it held, and the +// caller of every method, hooks are called with it held, and the // goroutines the router starts (pump driver, media prepares, drains and timer // callbacks) take it themselves. After unlocked I/O, revalidate the owning // room, operation and resource before committing effects under the lock. @@ -52,16 +52,16 @@ func routeDebugSink() func(string, ...any) { return slog.Info } -// sfuFallback is SfuFallbackOptions. A zero timeout means the TS default. +// sfuFallback supplies embedded media and admission; zero selects the default timeout. type sfuFallback struct { media sfu.Runtime admission *sfu.Admission prepareTimeoutMs int64 } -// routerHooks are the signaling server callbacks of HybridMediaRouterOptions. +// routerHooks are callbacks into the signaling server. // Every hook is called with mu held and must not block. shareGeneration -// returns "" where the TS returned undefined; routesChanged may be nil. +// returns "" when no share exists; routesChanged may be nil. type routerHooks struct { sendToSession func(sessionID string, message protocol.ServerMessage) shareGeneration func(roomID string) string @@ -81,8 +81,7 @@ type routerOptions struct { afterFunc func(time.Duration, func()) func() bool } -// authenticatedRouteParticipant is AuthenticatedRouteParticipant; routePolicy -// is nil where the TS key was absent. +// authenticatedRouteParticipant carries admitted identity and optional Host policy. type authenticatedRouteParticipant struct { roomID string role protocol.Role @@ -91,22 +90,21 @@ type authenticatedRouteParticipant struct { routePolicy *protocol.RoutePolicy } -// hybridAuthenticationState is HybridAuthenticationState. +// hybridAuthenticationState supplies the joining session's route assignment. type hybridAuthenticationState struct { routeRevision int64 routeAssignment protocol.ParticipantRouteAssignment } -// activeViewerMediaEdge is ActiveViewerMediaEdge; upstream is peer or sfu. +// activeViewerMediaEdge identifies one Viewer's active Peer or SFU upstream. type activeViewerMediaEdge struct { revision int64 connectionID string upstream protocol.MediaRouteUpstream } -// signalAuthorization is the `boolean | "probe" | undefined` result of -// peerSignalAuthorization. signalAuthorizationUnassigned is the TS undefined: -// the signaling server falls back to the committed-edge assignment. +// signalAuthorization is the peerSignalAuthorization result. Unassigned makes +// the signaling server check the committed-edge assignment. type signalAuthorization int const ( @@ -116,8 +114,8 @@ const ( signalAuthorizationProbe ) -// peerSignalInput is the peerSignalAuthorization input; descriptionType is -// "" where the TS key was absent. +// peerSignalInput supplies session/connection identity for signal authorization. +// descriptionType is empty for signals without an SDP description. type peerSignalInput struct { roomID string sourcePeerID string @@ -129,8 +127,7 @@ type peerSignalInput struct { descriptionType string } -// peerSignalDebugInput is the debugPeerSignal input; candidateOrigin and -// descriptionType are "" where the TS keys were absent. +// peerSignalDebugInput supplies signal provenance; optional fields are empty. type peerSignalDebugInput struct { roomID string sourcePeerID string @@ -141,8 +138,8 @@ type peerSignalDebugInput struct { authorization signalAuthorization } -// roomRuntime is RoomRuntime. pumping is the `pump` promise used as a -// single-flight token; deadlineGeneration is bumped by every clearDeadline so +// roomRuntime owns one controller. pumping prevents concurrent drivers; +// deadlineGeneration is bumped by every clearDeadline so // a callback that lost the Stop race sees it is stale. type roomRuntime struct { hostPeerID string @@ -155,7 +152,7 @@ type roomRuntime struct { deadlineGeneration uint64 } -// router is HybridMediaRouter. +// router applies controller decisions through room signaling and media resources. type router struct { mu *sync.Mutex store *room.Store @@ -172,13 +169,12 @@ type router struct { resourceWaiters ordered.Map[string, struct{}] sfuDrainTasks ordered.Map[sfu.SubscriptionFence, *sfuDrainTask] // inflight counts the goroutines that will re-acquire mu: the pump - // drivers, fresh SFU config issues and drains. It is the TS microtask - // backlog; tests wait for it to reach zero where the TS awaited. + // drivers, fresh SFU config issues and drains. inflight int closing bool } -// newRouter is the HybridMediaRouter constructor. +// newRouter shares the server's lock and supplies clock defaults. func newRouter(options routerOptions) *router { if err := protocol.AssertEndpointMediaCopyCapacity(options.endpointMediaCopyCapacity); err != nil { panic(err) @@ -205,12 +201,9 @@ func newRouter(options routerOptions) *router { return r } -// io is one TS `await` on external work: mu is released around fn and always -// re-acquired, including while a panic unwinds. That last part is the reason -// this is a helper rather than an inline Unlock/Lock pair: recoverPump turns a -// panic raised inside such a window into the TS pump rejection, and its -// caller's `defer r.mu.Unlock()` would otherwise unlock a mutex nobody holds -// (an unrecoverable runtime error). Nothing may touch router state inside fn. +// io releases mu for external work and reacquires it even while a panic unwinds. +// This keeps the caller's deferred unlock valid when recoverPump catches the +// failure. Nothing may touch router state inside fn. func (r *router) io(fn func()) { r.mu.Unlock() defer r.mu.Lock() @@ -266,9 +259,7 @@ func (r *router) connectParticipant(input authenticatedRouteParticipant) hybridA if rm.controller != nil && rm.hostPeerID != input.peerID { r.clearDeadline(rm) // rm.hostPeerID always equals the controller's, so this branch - // takes RebindHostIdentity's clearing path: every committed edge - // goes, which is what the TS per-viewer deleteConnectionId loop - // then restated into the mirror. + // takes RebindHostIdentity's clearing path: every committed edge goes. r.releaseResources(rm.controller.RebindHostIdentity(input.peerID, input.sessionID, r.nowPtr())) } rm.hostPeerID = input.peerID @@ -1089,7 +1080,7 @@ func (r *router) assignments( publicationGeneration = snapshot.HostPublication.Generation } if candidate != nil && candidateChildPeerID != "" { - // TS delete-then-set: the candidate child moves to the back. + // The replacement candidate follows the surviving committed children. edges.Delete(candidateChildPeerID) if candidate.Kind == route.UpstreamPeer { edges.Set(candidateChildPeerID, assignmentEdge{kind: route.UpstreamPeer, parentPeerID: candidate.ParentPeerID}) @@ -1292,8 +1283,8 @@ type qualitySample struct { hostSessionID string } -// qualityCopyContext is QualityCopyContext; operationReason and -// candidateTransition are "" where the TS had null. +// qualityCopyContext attributes copy demand to committed and candidate work; +// operationReason and candidateTransition are empty when no such work exists. type qualityCopyContext struct { committedCopies int candidateReservedCopies int @@ -1519,8 +1510,7 @@ func nullable[T any](value protocol.Nullable[T]) any { return *value.Value } -// nullString is the debug value of a string that is "" where the TS had -// null or undefined. +// nullString emits absent debug identities as null. func nullString(value string) any { if value == "" { return nil diff --git a/internal/server/signal/router_pump.go b/internal/server/signal/router_pump.go index 3b4b8c1e6..7a93ba413 100644 --- a/internal/server/signal/router_pump.go +++ b/internal/server/signal/router_pump.go @@ -12,7 +12,7 @@ import ( "github.com/TNTcraftHIM/Piik/internal/server/sfu" ) -// preparedCandidate is PreparedCandidate; the optional TS keys are "" / nil. +// preparedCandidate holds the allocations and signaling needed to begin an attempt. type preparedCandidate struct { reservation route.CandidateReservation connectionID string @@ -446,9 +446,8 @@ func (r *router) sendPrepareMessages( } } -// scheduleDeadline registers a callback that verifies it is -// still the registered deadline through the room's generation counter, which -// clearDeadline bumps: a cleared TS timer never fired. +// scheduleDeadline fences callbacks by the room's deadline generation. +// clearDeadline invalidates even a callback that already won the timer Stop race. func (r *router) scheduleDeadline(roomID string, rm *roomRuntime, operation *route.OperationSnapshot) { r.clearDeadline(rm) wakeInMs := max(int64(0), operation.WakeAtMs-r.now()) diff --git a/internal/server/signal/server.go b/internal/server/signal/server.go index 51d7b766f..ca0fc8c25 100644 --- a/internal/server/signal/server.go +++ b/internal/server/signal/server.go @@ -36,19 +36,17 @@ import ( // Connection admission, queue and timeout bounds limit the signaling resources // a stalled or unauthenticated peer can retain. const ( - // maxBufferedSignalBytes is MAX_BUFFERED_SIGNAL_BYTES: once a connection's - // unsent queue passes it the peer is not draining and the connection is - // terminated, so one stalled reader cannot grow the server's heap. + // maxBufferedSignalBytes bounds unsent data per connection; exceeding it + // terminates a stalled reader before it can grow the server's heap. maxBufferedSignalBytes = 256 * 1024 // defaultMaxSignalConnections and defaultMaxUnauthenticatedConnections are - // the DEFAULT_MAX_* admission ceilings: 20 viewers plus a host across 9_000 + // admission ceilings: 20 viewers plus a host across 9_000 // room codes is far above either, so they bound abuse, not capacity. defaultMaxSignalConnections = 2_048 defaultMaxUnauthenticatedConnections = 256 - // minSignalingChallengeIntervalMs is MIN_SIGNALING_CHALLENGE_INTERVAL_MS: - // the floor between two liveness challenges on one connection. + // minSignalingChallengeIntervalMs bounds liveness challenge frequency. minSignalingChallengeIntervalMs = int64(1_000) - // serviceRestartCloseGrace is SERVICE_RESTART_CLOSE_GRACE_MS: how long a + // serviceRestartCloseGrace is how long a // peer has to acknowledge close{1012} before the socket is terminated. serviceRestartCloseGrace = 1_000 * time.Millisecond @@ -97,12 +95,12 @@ type Options struct { MaxUnauthenticatedConnections int // AfterFunc is the timer factory; nil uses time.AfterFunc. AfterFunc func(time.Duration, func()) func() bool - // Logger receives the three console.error records; nil uses slog.Default(). + // Logger receives signaling failures; nil uses slog.Default(). Logger *slog.Logger } -// viewerQualityEvidenceGate is ViewerQualityEvidenceGate; upstreamPeerID is -// "" where the TS had null. +// viewerQualityEvidenceGate rejects stale or repeated evidence for the current +// edge and presentation epoch. upstreamPeerID is empty for SFU. type viewerQualityEvidenceGate struct { viewerSessionID string upstreamKind string @@ -112,32 +110,16 @@ type viewerQualityEvidenceGate struct { sequence int64 } -// viewerQualityEvidenceAttempt is viewerQualityEvidenceAttemptAtMs plus the -// committed-edge connection the stamp was taken under. Tagging replaces the -// TS setViewerConnectionId reset: a viewer whose media identity changed is -// never held back by the previous route's stamp. -// -// Deviation: it does not replace the deleteViewerConnectionId reset, which -// broadcastActive also ran when a viewer's path stopped being physical. A -// viewer whose path breaks and heals on the same committed connection without -// sending evidence in between keeps its stamp here, where the TypeScript had -// cleared it, so one frame inside the 2 s window can be dropped that the -// TypeScript accepted. Unobservable: the reporter's own -// VIEWER_QUALITY_EVIDENCE_INTERVAL_MS gate means its next send is already at -// least that far from the last accepted one. The epoch/sequence gate below -// cannot be affected the same way, because a committed edge that loses -// physicalActive is never reactivated - its replacement carries a new -// connectionID, so sameEdge is false. +// viewerQualityEvidenceAttempt scopes the rate window to a committed connection. +// A replacement connection is not held back by its predecessor's timestamp. +// Reports for the same connection remain subject to the existing window. type viewerQualityEvidenceAttempt struct { connectionID string atMs int64 } -// roomShare is one Host share of one room. Its zero value is the TS state -// where none of the four per-room share maps had an entry, so reads need no -// presence check: generation and pausedGeneration are never stored empty -// (authenticate falls back to the session id), which is what made the TS -// `undefined === undefined` comparisons meaningful. +// roomShare owns one room's Host publication intent. Its zero value means no +// intent; active generations are nonempty, falling back to the Host session ID. type roomShare struct { // generation survives stopSharing; only closeRoom drops the whole // record. @@ -150,19 +132,18 @@ type roomShare struct { pausedGeneration string } -// senderQualityRateWindow is SenderQualityRateWindow. type senderQualityRateWindow struct { startedAtMs int64 count int } -// graceTimer is one viewerGraceTimers entry; the callback compares the -// registered pointer with its own (a cleared TS timer never fired). +// graceTimer callbacks compare their registered pointer so replacement or +// cancellation invalidates a callback even after its timer has fired. type graceTimer struct { stop func() bool } -// Server is SignalingServer. +// Server owns room signaling and serializes its effects. type Server struct { // ponytail: global lock, per-room locks if throughput matters mu sync.Mutex @@ -200,9 +181,7 @@ type Server struct { viewerQualityEvidenceGates map[string]viewerQualityEvidenceGate viewerQualityEvidenceAttempts map[string]viewerQualityEvidenceAttempt senderQualityRateBySession map[string]*senderQualityRateWindow - // shares owns the whole per-room share lifecycle (TS - // shareGenerationsByRoom, qualitySettingsByRoom, routePolicyByRoom and - // pausedShareGenerationsByRoom). + // shares keeps generation, settings, route policy and pause intent together. shares map[string]roomShare deferredViewerPresenceRooms map[string]struct{} @@ -210,8 +189,7 @@ type Server struct { closing bool } -// New is the SignalingServer constructor. It performs no I/O; the heartbeat -// ticker starts immediately. +// New creates the signaling owner and starts its heartbeat timer. func New(options Options) (*Server, error) { if err := protocol.AssertEndpointMediaCopyCapacity(options.EndpointMediaCopyCapacity); err != nil { return nil, err @@ -318,9 +296,8 @@ func orDefault(value, fallback int) int { return value } -// InviteURL is viewerInviteUrl / the inviteUrl of createRoomResponse: -// new URL(`/r/${roomId}`, publicBaseUrl) with the Viewer grant in the -// fragment only. viewerGrant "" is the TypeScript null. +// InviteURL adds an optional Viewer grant only in the URL fragment, keeping it +// out of HTTP requests. Existing base URL query/fragment values are discarded. func InviteURL(publicBaseURL *url.URL, roomID, viewerGrant string) string { invite := *publicBaseURL invite.Opaque = "" @@ -340,20 +317,16 @@ func InviteURL(publicBaseURL *url.URL, roomID, viewerGrant string) string { // upgrade // --------------------------------------------------------------------------- -// ServeHTTP is the "upgrade" listener: the TS rejection ladder, then the -// accept. +// ServeHTTP validates the signaling endpoint, origin and capacity before upgrade. func (s *Server) ServeHTTP(writer http.ResponseWriter, request *http.Request) { - // accepts a scheme-relative target such as "//[" as a plain path, so - // the WHATWG authority parse is re-run on the raw target. + // HTTP parsing accepts scheme-relative targets as paths; reject malformed + // authorities such as "//[" before checking the endpoint. if _, err := url.Parse(request.RequestURI); err != nil { rejectUpgrade(writer, http.StatusBadRequest) return } - // EscapedPath, not Path: the WHATWG pathname the TS compared is not - // percent-decoded, so "/%73ignal" was a 404 there, while URL.Path decodes - // it to "/signal" and would give the endpoint an alias the TS never had. - // This is the path notion the sibling HTTP router already uses - // (app/json.go requestPath), with the same dot-segment deviation. + // Match the literal endpoint, rejecting encoded aliases such as "/%73ignal", + // just as the sibling HTTP router's requestPath does. if request.URL.EscapedPath() != "/signal" || request.URL.RawQuery != "" { rejectUpgrade(writer, http.StatusNotFound) return @@ -374,15 +347,14 @@ func (s *Server) ServeHTTP(writer http.ResponseWriter, request *http.Request) { rejectUpgrade(writer, http.StatusServiceUnavailable) return } - // The TS counted the slot synchronously inside handleUpgrade; Accept - // performs I/O, so the slot is reserved across it. + // Reserve capacity before Accept performs I/O outside the lock. s.pendingConnections++ s.mu.Unlock() siteAccessAuthenticated := s.siteAccessAtUpgrade(request) var sess *session // Origin was verified above with the exact isAllowedOrigin rule, so the - // library's own check is skipped; the TS ws server never compressed. + // library's own check is skipped. conn, err := websocket.Accept(writer, request, &websocket.AcceptOptions{ InsecureSkipVerify: true, CompressionMode: websocket.CompressionDisabled, @@ -572,10 +544,9 @@ func (s *Server) UpdateRoomAccess( return nil, fmt.Errorf("unknown room access request %T", request) } -// deriveViewerPasswordMaterial is the unlocked KDF window of replaceRoom and -// setViewerPassword; mayStart is the TS `rooms.get(roomId) === current && -// verifyDigest(hostToken)` closure. mu must be held; it is released only -// while the derivation runs. A nil password derives nothing. +// deriveViewerPasswordMaterial releases mu for password derivation, rechecking +// exact room/Host authority when the gate admits the work. The caller holds mu; +// a nil or empty password derives nothing. func (s *Server) deriveViewerPasswordMaterial( roomID, hostToken string, password *string, current *room.Room, ) (material []byte, derived error) { @@ -619,8 +590,8 @@ func (s *Server) ReplaceRoom( return replacement.Created, nil } -// CreateRoom is the roomStore.createRoom call of app.ts, run under mu with -// the KDF unlocked (the room package's documented sequence). +// CreateRoom serializes room admission and commit, releasing mu for password +// derivation according to the room package's locking contract. func (s *Server) CreateRoom( codeEntryPolicy protocol.CodeEntryPolicy, roomPassword *string, @@ -907,7 +878,7 @@ func (s *Server) authenticationFailed(sess *session, request authRequest, err er func (s *Server) completeAuthentication(sess *session, request authRequest, participant room.ConnectedParticipant) { if s.closing || !s.sessions.Has(sess) || sess.revoked || !sess.open() { if _, err := s.store.DisconnectParticipant(participant.RoomID, participant.PeerID, sess.sessionID); err != nil { - // The TS threw out of the async function; there is no catcher. + // Failing to retire the aborted admission is an internal invariant failure. panic(fmt.Errorf("room store failed while releasing an aborted authentication: %w", err)) } return @@ -1093,8 +1064,7 @@ func (s *Server) settleHostShare(roomID, sessionID string, request authRequest) if currentGeneration != "" && currentGeneration != shareGeneration { s.stopSharing(roomID) } - // stopSharing may have cleared the record, so the survivors are read - // after it, exactly as the TS map reads were. + // stopSharing may have cleared fields, so read the surviving intent afterwards. share := s.shares[roomID] if request.shareGeneration != "" && (currentGeneration != shareGeneration || share.qualitySettings == nil) { @@ -1150,9 +1120,8 @@ func authenticatedDisplayName(request authRequest) *string { return &name } -// connectRouteParticipant is the try/catch around -// hybridMediaRouter.connectParticipant: a controller assertion panics with -// the TS message, which the TS catch swallowed. +// connectRouteParticipant leaves the participant unassigned if a controller +// assertion prevents route admission. func (s *Server) connectRouteParticipant(participant authenticatedRouteParticipant) (state hybridAuthenticationState, assigned bool) { defer func() { if recover() != nil { @@ -1471,7 +1440,7 @@ func (s *Server) handleViewerQualityEvidence(sess *session, source *authenticate } // The committed edge owns the viewer's media identity; connectionID is - // "" while the viewer has no physical path (the TS mirror was absent). + // empty while the viewer has no physical path. edge, hasEdge := s.router.resolveActiveViewerMediaEdge(source.roomID, source.peerID) connectionID := "" if hasEdge { @@ -1701,8 +1670,7 @@ func (s *Server) routePeerAssistedSignal(sess *session, source *authenticatedSes return } - // TS setViewerConnectionId here only ever restated the committed edge: - // an authorized non-probe offer either matches edge.connectionId or was + // An authorized non-probe offer either matches the committed connection or was // adopted into it by peerSignalAuthorization. s.sendToSession(target.SessionID, protocol.ServerSignalMessage{ Type: "signal", diff --git a/internal/server/signal/session.go b/internal/server/signal/session.go index e6c169ce8..ca4585ec9 100644 --- a/internal/server/signal/session.go +++ b/internal/server/signal/session.go @@ -35,14 +35,13 @@ type outbound struct { reason string } -// session is SocketState plus the connection it belongs to. Every field is +// session owns one signaling connection and its admission state. Every field is // guarded by Server.mu except conn, ctx, cancel, wake and readerDone, which // are immutable after accept. type session struct { conn *websocket.Conn // ctx is cancelled by terminate(); coder/websocket closes the socket - // when the context of a pending Read or Write expires, which is the - // `socket.terminate()` of the TS. + // when the context of a pending Read or Write expires. ctx context.Context cancel context.CancelFunc @@ -51,7 +50,7 @@ type session struct { siteAccessAuthenticated bool revoked bool authenticating bool - // challenged is `lastSignalingChallengeAtMs !== undefined`. + // challenged distinguishes no challenge yet from a timestamp of zero. challenged bool lastSignalingChallengeAtMs int64 authenticated *authenticatedSession @@ -59,17 +58,17 @@ type session struct { nextInteractionAtMs int64 // authStop stops the authentication deadline; authGeneration is - // bumped whenever the TS cleared the timer so a callback that lost the + // bumped whenever the timer is cleared so a callback that lost the // Stop race sees it is stale. authStop func() bool authGeneration uint64 - // queue is the ws send buffer; queuedBytes is its bufferedAmount. + // queuedBytes accounts for pending outbound text frames. queue []outbound queuedBytes int wake chan struct{} - // closeQueued is readyState !== OPEN once close() was called: later - // sends are dropped and later closes are no-ops, as with ws. + // closeQueued stops further sends and repeated close requests while the + // writer completes the existing queue and close handshake. closeQueued bool terminated bool readerDone chan struct{} @@ -89,7 +88,7 @@ func newSession(conn *websocket.Conn, siteAccessAuthenticated bool) *session { } } -// open is `socket.readyState === WebSocket.OPEN`. +// open reports whether this session still accepts outgoing messages. func (sess *session) open() bool { return !sess.closeQueued && !sess.terminated } @@ -108,7 +107,7 @@ func (sess *session) enqueue(encoded []byte) { sess.notify() } -// close is socket.close(code, reason): a no-op unless the socket is open. +// close queues a close handshake only while the socket is open. // mu must be held. func (sess *session) close(code websocket.StatusCode, reason string) { if !sess.open() { @@ -119,7 +118,7 @@ func (sess *session) close(code websocket.StatusCode, reason string) { sess.notify() } -// terminate is socket.terminate(): abrupt, no close frame. Cancelling the +// terminate closes abruptly without a close frame. Cancelling the // context closes the socket from inside the library, so nothing touches the // connection under mu. mu must be held. func (sess *session) terminate() { @@ -130,7 +129,7 @@ func (sess *session) terminate() { sess.cancel() } -// clearAuthenticationTimer is clearTimeout(state.authenticationTimer). +// clearAuthenticationTimer invalidates pending and already-fired callbacks. func (sess *session) clearAuthenticationTimer() { if sess.authStop != nil { sess.authStop() @@ -155,8 +154,8 @@ func (s *Server) accept(conn *websocket.Conn, siteAccessAuthenticated bool) *ses return sess } -// armAuthenticationTimer is the setTimeout of accept(). The TS timer -// fired blind; the generation check stands in for clearTimeout. +// armAuthenticationTimer bounds admission time. The generation check rejects +// a stale callback even when stopping the timer lost the race. func (s *Server) armAuthenticationTimer(sess *session) { generation := sess.authGeneration sess.authStop = s.afterFunc(time.Duration(s.authenticationTimeoutMs)*time.Millisecond, func() { From 5084cca95a497a7ea49491cdcdafd5ff757c30c4 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:10:42 +0800 Subject: [PATCH 024/262] docs(todo): include thumbnail retirement in capture border review --- docs/todo.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/todo.md b/docs/todo.md index adc4292d5..eb2da4b06 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -17,8 +17,9 @@ Git/PRs own completed history. before accepting a changed contract. - [ ] **Windows 11 capture border remains visible.** Reopened by a new report on 2026-10-07. Recheck App/Browser capture ownership, border permission/API - results, live setting changes and concurrent captures before attributing this - to Windows. Obtain the affected version/path/build when available; the report + results, live setting changes, concurrent captures and source-thumbnail + cancellation/retirement before attributing this to Windows. Obtain the + affected version/path/build when available; the report does not yet identify them. [Capture evidence](./research/native-client-lifecycle.md#windows-capture-borders) owns the prior graceful-retirement and concurrent-capture checks. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance From 9079c7c5050a443333b1d7cfda9674be1347d2c4 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:19:05 +0800 Subject: [PATCH 025/262] docs(transport): reconcile isolated Magicsock acceptance evidence --- docs/research/nat-traversal.md | 71 +++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 10 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index d7f6e1e5a..968c0a723 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1,6 +1,6 @@ # Browser And Native NAT Traversal -Last reviewed: 2026-10-04 +Last reviewed: 2026-10-07 This document owns evidence for improving direct ICE without adding a new relay or a custom transport. The current product contract remains standard @@ -319,8 +319,8 @@ isolated branch; production dependencies and routing remain unchanged. ### Integration Cost And Recovery Controls -The 2026-10-06 assessment retains the probes on `spike/magicsock-assessment` -at `fea0587f`, based on current main. These nested Go modules are outside Piik's +The isolated `spike/magicsock-assessment` branch retains the comparison probes. +These nested Go modules are outside Piik's build and dependency graph. Repeated local Tailcat bootstrap, peer denial, revocation and synthetic SRTP media checks passed; the raw magicsock direct-path matrix again found the same reachable classes as fresh Pion ICE. Retired shared @@ -353,13 +353,64 @@ increasing only the fixture's ICE budget recovered media. The ordinary Pion loss control recovered without that override. Do not adopt packet prefixes, diagnostic Ping polling or wider product deadlines as integration shortcuts. -Reassessed on 2026-10-07: keep the transport isolated. Go reuse is feasible, but -new key authorization/revocation, underlay readiness, packet-size limits and -relay operation have no demonstrated direct-coverage benefit sufficient for -product integration. A future trial needs matched networks with a repeatable -benefit, separate direct/relayed/first-media results, and current room/route -ownership. IPv6, gateway mapping, Piik prediction, physical routers, decoded -playback and mixed Browser/Native fanout remain outside this comparison. +Keep product integration conditional on a repeatable benefit on matched networks, +with direct coverage, relay coverage, recovery and first picture measured +separately. Go feasibility alone does not establish that benefit. Gateway +mapping, Piik prediction, physical routers and the full mixed Piik topology +remain outside the comparison. The isolated controls below do not change that +adoption boundary. + +### Isolated Connection-Layer Acceptance + +The 2026-10-07 experiment through `f0ca8056` keeps Pion's ICE/DTLS/SRTP over +Tailcat. Each physical connection owns a virtual UDP tuple and mux; its one-shot +listener retires after admission. Connections to one authorized peer may share +the tunnel. Closing or aborting one flow preserves healthy siblings; revoking +the fixture peer key denies admission and disconnects that tunnel. These checks +do not implement Piik room grants or endpoint exchange. + +Repeated lifecycle and Linux race checks cover overlapping media preparation, +rollback, replacement, canceled preparation followed by retry, bounded datagrams +and blocked-reader retirement. A 350 ms interruption of one flow preserved its +sibling and recovered without replacing SDP. Piik's actual encoded source, +LiveKit forwarder and pacer also carried a small VP8/Opus fixture over ordinary +Pion and Tailcat, including TWCC and keyframe requests. Tailcat recovered an +injected missing video packet through existing NACK handling after DERP closed. +This is forwarding evidence, not throughput or room-controller acceptance. + +A separate real Chrome control decoded H.264/Opus and VP8/Opus from a generated +640×360 picture and tone through Browser WebRTC → Pion/Tailcat/Pion → Browser +WebRTC without re-encoding. Picture and decoded audio continued after DERP +closure and stopped after peer revocation. It uses ordinary Browser WebRTC, +not Browser-native Magicsock, physical capture or cross-browser acceptance. + +The 1200-byte Native packetization envelope with TWCC and short MID/RID headers +produced at most 1228-byte encrypted datagrams across Pion's three default SRTP +profiles. Only four bytes remain below Tailcat's 1232-byte ceiling; larger +headers and maximum compound RTCP remain integration checks. The small actual +forwarder fixture does not establish those limits. + +Kernel gateway rebinding retained full WebRTC sessions. With DERP available, +fresh media resumed in about 6.4–6.5 seconds and ICE reported `connected` in about +7.1–8.1 seconds, without new SDP. With DERP closed, neither gateway direction +recovered within 20 seconds. Fresh media and ICE state are separate observations; +Piik's disconnected grace, restart and retirement need a full integration +control before adoption. Underlay preparation must fit the existing operation +deadline, not start a second budget. Local cold setup of about 3.3 seconds for +Tailcat versus 0.2 seconds for Pion disabled STUN/gateway mapping and is not a WAN +startup comparison. + +Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or +IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof +required fresh bidirectional delivery after DERP closed. One side reporting a +direct address did not establish that both directions were direct. IPv4 TCP +still supplied bootstrap, so this is not an IPv6-only or physical ISP check. +These controls add no demonstrated direct-reachability advantage for Magicsock. + +The experiment's README owns commands and fixture boundaries; raw output remains +ignored. Further product work needs a measurable network benefit and accepted +room authorization, resource and relay-operation owners, not more loopback +successes. ## Native Mapping Cache And Retry Lifetimes From 74bb1b8df3c0bd26eaf7f2f49b757b599f9bc15f Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:50:36 +0800 Subject: [PATCH 026/262] docs(transport): record feedback boundary for Magicsock adoption --- docs/research/nat-traversal.md | 16 ++++++++++++++++ docs/todo.md | 5 +++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 968c0a723..0a8555cd7 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -390,6 +390,22 @@ profiles. Only four bytes remain below Tailcat's 1232-byte ceiling; larger headers and maximum compound RTCP remain integration checks. The small actual forwarder fixture does not establish those limits. +A feedback control found a concrete mismatch: Pion's TWCC recorder generated +a 1244-byte encrypted report and a 1268-byte compound batch. Ordinary Pion +delivered both; the bounded adapter rejected them, and unwrapped Tailcat did +not deliver either within two seconds. A 1144-byte report arrived before and +after. Windows and Linux race checks repeated this result three times each. +Pion ICE treats non-close socket write errors as packet loss, so `WriteRTCP` +returning no error did not prove delivery. This does not establish how often +Piik would produce these batches. Tailcat's +[documented payload bound](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/tailcat.go#L574) +needs a supported feedback-size boundary before integration; reducing RTP +packetization alone does not bound RTCP. No product transport or custom packet +fragmentation was added for this check. +The related [Pion feedback-MTU report](https://github.com/pion/interceptor/issues/416) +remains open as of 2026-10-07. Its out-of-order packet evidence does not establish +the cause of any Piik field report; the pinned recorder has no byte-size option. + Kernel gateway rebinding retained full WebRTC sessions. With DERP available, fresh media resumed in about 6.4–6.5 seconds and ICE reported `connected` in about 7.1–8.1 seconds, without new SDP. With DERP closed, neither gateway direction diff --git a/docs/todo.md b/docs/todo.md index eb2da4b06..df9357014 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -60,8 +60,9 @@ Implementation needs a concrete useful result within the product model. [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) have not established improved direct reachability. Seek a repeatable benefit before accepting peer-key authorization/revocation, underlay readiness and - relay operation. Keep the isolated experiment separate from product routing; - the P2P evidence work below owns the network/interop acceptance boundary. + relay operation. The isolated feedback check also requires a supported RTCP + size boundary for the tunnel's MTU. Keep the experiment separate from product + routing; the P2P evidence work below owns the network/interop acceptance boundary. - [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) identifies UDP/storage requirements but establishes no cost saving. Keep the existing P2P-only Demo until a provider and operating boundary are accepted. From a9e8ceefc505cd6dcf3c2530dd100d87b25ee860 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:02:08 +0800 Subject: [PATCH 027/262] docs(capture): include display changes in border investigation --- docs/research/native-client-lifecycle.md | 6 ++++++ docs/todo.md | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/research/native-client-lifecycle.md b/docs/research/native-client-lifecycle.md index 4595b74e8..291a7adfb 100644 --- a/docs/research/native-client-lifecycle.md +++ b/docs/research/native-client-lifecycle.md @@ -343,6 +343,12 @@ This establishes a retirement defect and its local repair, not the cause of every reported Windows 11 border: denied consent and other active captures remain distinct Windows-owned limits. +The upstream [display-resolution change report](https://github.com/robmikh/Win32CaptureSample/issues/59) +is still open as of 2026-10-07: it reports borderless display capture losing that +preference after a mode change. It gives neither a current Windows build nor a +verified repair. Include display-mode changes in physical reproduction; this +report alone does not justify toggling the property or restarting capture. + An unofficial WGC-preserving option exists: [Windhawk's DWM Custom Projection Border](https://github.com/ramensoftware/windhawk-mods/blob/main/mods/dwm-custom-projection-border.wh.cpp) documents disabling the border on Windows 10 21H2. Source inspection confirms diff --git a/docs/todo.md b/docs/todo.md index df9357014..5752d56ff 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -17,9 +17,9 @@ Git/PRs own completed history. before accepting a changed contract. - [ ] **Windows 11 capture border remains visible.** Reopened by a new report on 2026-10-07. Recheck App/Browser capture ownership, border permission/API - results, live setting changes, concurrent captures and source-thumbnail - cancellation/retirement before attributing this to Windows. Obtain the - affected version/path/build when available; the report + results, live setting changes, display-mode changes, concurrent captures and + source-thumbnail cancellation/retirement before attributing this to Windows. + Obtain the affected version/path/build when available; the report does not yet identify them. [Capture evidence](./research/native-client-lifecycle.md#windows-capture-borders) owns the prior graceful-retirement and concurrent-capture checks. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance From eca1c6a373675f981b898e6ee0daf409ebafab13 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 06:03:56 +0800 Subject: [PATCH 028/262] fix(capture): retain signals for in-flight Windows callbacks Keep callback event handles alive through asynchronous WinRT dispatch in both capture and thumbnail paths. Remove the redundant stack-owned close flag and verify late callbacks cannot signal a replacement capture. --- native/capture/windows/capture_event.h | 19 ++++++++++ native/capture/windows/capture_target.cpp | 17 ++++----- .../capture/windows/capture_target.test.cpp | 31 ++++++++++++++++ native/capture/windows/main.cpp | 36 ++++++++----------- 4 files changed, 70 insertions(+), 33 deletions(-) create mode 100644 native/capture/windows/capture_event.h diff --git a/native/capture/windows/capture_event.h b/native/capture/windows/capture_event.h new file mode 100644 index 000000000..c89861685 --- /dev/null +++ b/native/capture/windows/capture_event.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +#include +#include + +namespace piik::capture { + +// Revocation does not join an in-flight WinRT callback. Let the delegate retain +// its signal until the final callback returns, without retaining the capture. +inline auto CaptureEventHandler(std::shared_ptr event) { + return [event = std::move(event)](const auto&, const auto&) { + SetEvent(event->get()); + }; +} + +} // namespace piik::capture diff --git a/native/capture/windows/capture_target.cpp b/native/capture/windows/capture_target.cpp index f9a789cc5..158a23823 100644 --- a/native/capture/windows/capture_target.cpp +++ b/native/capture/windows/capture_target.cpp @@ -18,6 +18,7 @@ #include "capture_target.h" #include "capture_border.h" +#include "capture_event.h" #include "capture_geometry.h" #include "capture_sdr.h" @@ -489,7 +490,8 @@ HRESULT CaptureWithWgc(TargetKind kind, UINT64 source_id, Direct3D11CaptureFramePool pool{nullptr}; GraphicsCaptureSession session{nullptr}; CaptureBorder border; - HANDLE frame_ready = CreateEventW(nullptr, FALSE, FALSE, nullptr); + auto frame_ready = std::make_shared( + CreateEventW(nullptr, FALSE, FALSE, nullptr)); winrt::event_token frame_token{}; bool subscribed = false; HRESULT result = S_OK; @@ -498,7 +500,7 @@ HRESULT CaptureWithWgc(TargetKind kind, UINT64 source_id, if (SUCCEEDED(result)) result = CreateWgcItem(kind, source_id, &item); if (SUCCEEDED(result)) { auto size = item.Size(); - if (size.Width <= 0 || size.Height <= 0 || frame_ready == nullptr) { + if (size.Width <= 0 || size.Height <= 0 || !*frame_ready) { result = HRESULT_FROM_WIN32(ERROR_INVALID_DATA); } else { windows::CaptureDisplayColor display_color(kind, source_id); @@ -508,15 +510,11 @@ HRESULT CaptureWithWgc(TargetKind kind, UINT64 source_id, capture_device, static_cast(color.Format()), 1, size); session = pool.CreateCaptureSession(item); - frame_token = pool.FrameArrived( - [frame_ready](const Direct3D11CaptureFramePool&, - const winrt::Windows::Foundation::IInspectable&) { - if (frame_ready != nullptr) SetEvent(frame_ready); - }); + frame_token = pool.FrameArrived(CaptureEventHandler(frame_ready)); subscribed = true; border.Start(session); session.StartCapture(); - const DWORD wait = WaitForSingleObject(frame_ready, 1'500); + const DWORD wait = WaitForSingleObject(frame_ready->get(), 1'500); border.Apply(session); if (wait != WAIT_OBJECT_0) { result = wait == WAIT_TIMEOUT @@ -603,9 +601,6 @@ HRESULT CaptureWithWgc(TargetKind kind, UINT64 source_id, capture_device = nullptr; context.Reset(); device.Reset(); - if (frame_ready != nullptr) { - CloseHandle(frame_ready); - } if (uninitialize) { CoUninitialize(); } diff --git a/native/capture/windows/capture_target.test.cpp b/native/capture/windows/capture_target.test.cpp index b3d9895cc..9f293fcc2 100644 --- a/native/capture/windows/capture_target.test.cpp +++ b/native/capture/windows/capture_target.test.cpp @@ -1,4 +1,8 @@ #include "capture_target.h" +#include "capture_event.h" + +#include +#include #include #include @@ -7,6 +11,32 @@ namespace { +void CheckInFlightCaptureEvent() { + using namespace winrt::Windows::Graphics::Capture; + using namespace winrt::Windows::Foundation; + // A dispatched delegate can outlive both revocation and the capture owner. + // Exercise the handler shared by streaming capture and source thumbnails. + for (const bool manual_reset : {false, true}) { + auto signal = std::make_shared( + CreateEventW(nullptr, manual_reset, FALSE, nullptr)); + assert(*signal); + const HANDLE handle = signal->get(); + std::weak_ptr lifetime = signal; + TypedEventHandler in_flight{ + piik::capture::CaptureEventHandler(signal)}; + signal.reset(); + assert(!lifetime.expired()); + + winrt::handle replacement{CreateEventW(nullptr, FALSE, FALSE, nullptr)}; + assert(replacement); + in_flight(nullptr, nullptr); + assert(WaitForSingleObject(handle, 0) == WAIT_OBJECT_0); + assert(WaitForSingleObject(replacement.get(), 0) == WAIT_TIMEOUT); + in_flight = nullptr; + assert(lifetime.expired()); + } +} + void CheckOwnedApplicationWindow() { HWND owner = CreateWindowExW(0, L"STATIC", L"Piik enumeration owner", WS_OVERLAPPEDWINDOW, 0, 0, 320, 180, nullptr, nullptr, nullptr, nullptr); @@ -57,6 +87,7 @@ int wmain(int count, wchar_t** arguments) { }; return result == 0 && listed(arguments[2]) && !listed(arguments[3]) ? 0 : 1; } + CheckInFlightCaptureEvent(); CheckOwnedApplicationWindow(); // Enumeration and preview are separate operations. A once-listed window may // become hidden before WGC creates its item. Unsupported capture may return diff --git a/native/capture/windows/main.cpp b/native/capture/windows/main.cpp index 839e6b3d5..640a40c6c 100644 --- a/native/capture/windows/main.cpp +++ b/native/capture/windows/main.cpp @@ -31,6 +31,7 @@ #include "capture_target.h" #include "capture_border.h" +#include "capture_event.h" #include "capture_geometry.h" #include "capture_color.h" #include "capture_sdr.h" @@ -1855,24 +1856,17 @@ void RunVideoCapture(ProductArguments arguments) { EnableFastCaptureUpdates(capture_session); piik::capture::CaptureBorder border; - UniqueHandle shutdown(CreateEventW(nullptr, TRUE, FALSE, nullptr)); - UniqueHandle frame_ready(CreateEventW(nullptr, FALSE, FALSE, nullptr)); - if (shutdown.get() == nullptr || frame_ready.get() == nullptr) { + auto shutdown = std::make_shared( + CreateEventW(nullptr, TRUE, FALSE, nullptr)); + auto frame_ready = std::make_shared( + CreateEventW(nullptr, FALSE, FALSE, nullptr)); + if (!*shutdown || !*frame_ready) { Check(HRESULT_FROM_WIN32(GetLastError()), "capture-events"); } - std::atomic item_closed{false}; auto frame_token = pool.FrameArrived( - [event = frame_ready.get()](const Direct3D11CaptureFramePool&, - const winrt::Windows::Foundation::IInspectable&) { - SetEvent(event); - }); - auto closed_token = item.Closed( - [event = shutdown.get(), &item_closed](const GraphicsCaptureItem&, - const winrt::Windows::Foundation::IInspectable&) { - item_closed.store(true); - SetEvent(event); - }); + piik::capture::CaptureEventHandler(frame_ready)); + auto closed_token = item.Closed(piik::capture::CaptureEventHandler(shutdown)); std::vector> workers; std::atomic active_status_written{false}; std::mutex failure_mutex; @@ -1882,10 +1876,10 @@ void RunVideoCapture(ProductArguments arguments) { std::lock_guard lock(failure_mutex); if (!failure) failure = error; } - SetEvent(shutdown.get()); + SetEvent(shutdown->get()); }; auto cleanup = [&]() noexcept { - SetEvent(shutdown.get()); + SetEvent(shutdown->get()); writer.Stop(); for (auto& worker : workers) worker->Stop(); try { @@ -1942,8 +1936,8 @@ void RunVideoCapture(ProductArguments arguments) { refresh_input = false; }; const HANDLE window_waits[] = { - process.get(), shutdown.get(), frame_ready.get()}; - const HANDLE display_waits[] = {shutdown.get(), frame_ready.get()}; + process.get(), shutdown->get(), frame_ready->get()}; + const HANDLE display_waits[] = {shutdown->get(), frame_ready->get()}; bool source_was_minimized = false; for (;;) { if (window_target && IsIconic(reinterpret_cast( @@ -1976,10 +1970,8 @@ void RunVideoCapture(ProductArguments arguments) { std::lock_guard lock(failure_mutex); if (failure) std::rethrow_exception(failure); } - if (item_closed.load()) { - Fail("capture-closed", "selected source stopped capture"); - } - Fail("capture-stopped", "source capture stopped"); + // A worker failure was handled above; the other live wake is item.Closed. + Fail("capture-closed", "selected source stopped capture"); } if (wait == WAIT_TIMEOUT) { // An explicit output/control request may reuse a quiet source; silence is not EOF. From 21afb4decee255483e146756d32cbb5d75ce7fd0 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 06:09:58 +0800 Subject: [PATCH 029/262] fix(capture): queue Linux failures on the pipeline bus Keep startup failures pending until the main loop dispatches them. Use the existing GStreamer bus lifetime instead of a callback that can run before startup or outlive shutdown. --- native/capture/linux/main.c | 17 +++++++++-------- native/capture/linux/main.test.c | 19 ++++++++++++++++++- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/native/capture/linux/main.c b/native/capture/linux/main.c index 855140635..dd6c75427 100644 --- a/native/capture/linux/main.c +++ b/native/capture/linux/main.c @@ -596,18 +596,16 @@ static NalSummary inspect_h264(const guint8 *data, gsize size) { return summary; } -static gboolean quit_loop(gpointer data) { - CaptureRun *run = data; - g_main_loop_quit(run->loop); - return G_SOURCE_REMOVE; -} - static void fail_run(CaptureRun *run, const char *message) { g_mutex_lock(&run->lock); gboolean first = run->failure == NULL; if (first) run->failure = g_strdup(message); g_mutex_unlock(&run->lock); - if (first) g_main_context_invoke(NULL, quit_loop, run); + // MainContext.invoke can execute immediately before the loop starts, losing + // quit(). The pipeline bus queues the failure and owns its callback lifetime. + if (first) gst_element_post_message(run->pipeline, + gst_message_new_application(GST_OBJECT(run->pipeline), + gst_structure_new_empty("piik-capture-failed"))); } static void fail_output(VideoOutput *output, const char *message) { @@ -895,7 +893,10 @@ static GstPadProbeReturn output_input_frame(GstPad *pad, GstPadProbeInfo *info, static gboolean bus_message(GstBus *bus, GstMessage *message, gpointer data) { (void)bus; CaptureRun *run = data; - if (GST_MESSAGE_TYPE(message) == GST_MESSAGE_ERROR) { + if (GST_MESSAGE_TYPE(message) == GST_MESSAGE_APPLICATION && + gst_message_has_name(message, "piik-capture-failed")) { + g_main_loop_quit(run->loop); + } else if (GST_MESSAGE_TYPE(message) == GST_MESSAGE_ERROR) { GError *error = NULL; char *debug = NULL; gst_message_parse_error(message, &error, &debug); diff --git a/native/capture/linux/main.test.c b/native/capture/linux/main.test.c index 591b59e84..142a04748 100644 --- a/native/capture/linux/main.test.c +++ b/native/capture/linux/main.test.c @@ -62,8 +62,15 @@ static void check_encoder_admission_and_control(void) { gst_object_unref(encoder); } +static gboolean capture_failure_timeout(gpointer data) { + (void)data; + g_error("capture failure was lost before the main loop started"); + return G_SOURCE_REMOVE; +} + static void check_bus_error_retains_primary_cause(void) { - CaptureRun run = {.loop = g_main_loop_new(NULL, FALSE), .output_count = 1}; + CaptureRun run = {.loop = g_main_loop_new(NULL, FALSE), .output_count = 1, + .pipeline = gst_pipeline_new(NULL)}; g_mutex_init(&run.lock); run.outputs[0].run = &run; run.outputs[0].encoder = gst_element_factory_make("identity", NULL); @@ -85,9 +92,19 @@ static void check_bus_error_retains_primary_cause(void) { unlink(frame_path); g_free(frame_path); g_assert_cmpstr(run.failure, ==, error->message); + // Startup callbacks can fail before the bus watch/main loop is running. + // The recorded failure must still retire the pipeline once dispatch starts. + GstBus *bus = gst_element_get_bus(run.pipeline); + guint watch = gst_bus_add_watch(bus, bus_message, &run); + guint timeout = g_timeout_add(500, capture_failure_timeout, NULL); + g_main_loop_run(run.loop); + g_source_remove(timeout); + g_source_remove(watch); + gst_object_unref(bus); gst_message_unref(message); g_error_free(error); gst_object_unref(run.outputs[0].encoder); + gst_object_unref(run.pipeline); g_free(run.failure); g_main_loop_unref(run.loop); g_mutex_clear(&run.lock); From f9d3d450f760313b8277c3026bd04caf313a45d7 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 06:26:30 +0800 Subject: [PATCH 030/262] docs(server): explain current HTTP and service ownership --- internal/server/app/access.go | 16 +++++-------- internal/server/app/app.go | 45 ++++++++++++++--------------------- internal/server/app/json.go | 26 +++++++------------- internal/server/app/routes.go | 29 ++++++++-------------- internal/server/app/static.go | 24 +++++++------------ 5 files changed, 50 insertions(+), 90 deletions(-) diff --git a/internal/server/app/access.go b/internal/server/app/access.go index f2a2b1ec8..ae5451052 100644 --- a/internal/server/app/access.go +++ b/internal/server/app/access.go @@ -68,7 +68,7 @@ func (s *Server) siteAccessForRequest(request *http.Request) *siteAccessGate { return &access } -// newSiteAccess mirrors the SiteAccess constructor, including its TTL check. +// newSiteAccess creates one process-scoped cookie authority. func newSiteAccess(options siteAccessOptions) (*siteAccessGate, error) { ttlSeconds := options.TTLSeconds if ttlSeconds == 0 { @@ -100,13 +100,12 @@ func (a *siteAccessGate) required() bool { return a.password != "" } -// passwordMatches mirrors passwordMatches: an unset password accepts anything, -// a set password needs a non-empty exact match. +// An unset password permits entry; a configured password requires an exact match. func (a *siteAccessGate) passwordMatches(value string) bool { return !a.required() || (value != "" && secretMatches(value, a.password)) } -// isAuthenticated mirrors isAuthenticated over a raw Cookie request header. +// isAuthenticated verifies the cookie signature and expiry when access is required. func (a *siteAccessGate) isAuthenticated(cookieHeader string) bool { if !a.required() { return true @@ -134,8 +133,7 @@ func (a *siteAccessGate) isAuthenticated(cookieHeader string) bool { return secretMatches(segments[2], a.sign(segments[0]+"."+segments[1])) } -// createCookie mirrors createCookie and returns the Set-Cookie header value, -// or "" where TS returned undefined (site access is not required). +// createCookie returns a Set-Cookie value, or "" when access is not required. func (a *siteAccessGate) createCookie() string { if !a.required() { return "" @@ -143,7 +141,6 @@ func (a *siteAccessGate) createCookie() string { expiresAt := floorSeconds(a.now()) + int64(a.ttlSeconds) payload := cookieVersion + "." + strconv.FormatInt(expiresAt, 10) value := payload + "." + a.sign(payload) - // Attribute order is the TS array order; the app writes it verbatim. attributes := []string{ a.cookieName() + "=" + value, "Path=/", @@ -180,8 +177,7 @@ func floorSeconds(milliseconds int64) int64 { return seconds } -// readCookie mirrors the TS reader: the first part whose trimmed name matches -// wins, and an empty value counts as absent. +// readCookie uses the first matching name; an empty value counts as absent. func readCookie(header string, name string) string { for _, part := range strings.Split(header, ";") { separator := strings.IndexByte(part, '=') @@ -193,7 +189,7 @@ func readCookie(header string, name string) string { } // secretMatches compares through SHA-256 digests so the comparison is constant -// time regardless of length, exactly as the TS helper does. +// time regardless of length. func secretMatches(actual string, expected string) bool { actualDigest := sha256.Sum256([]byte(actual)) expectedDigest := sha256.Sum256([]byte(expected)) diff --git a/internal/server/app/app.go b/internal/server/app/app.go index 1255087f0..4f3e75b47 100644 --- a/internal/server/app/app.go +++ b/internal/server/app/app.go @@ -41,21 +41,19 @@ import ( // HTTP request, header and keep-alive deadlines. const ( - requestTimeout = 10 * time.Second // httpServer.requestTimeout - headersTimeout = 15 * time.Second // httpServer.headersTimeout - keepAliveTimeout = 5 * time.Second // httpServer.keepAliveTimeout + requestTimeout = 10 * time.Second + headersTimeout = 15 * time.Second + keepAliveTimeout = 5 * time.Second ) -// Options is CreateServerOptions. Every zero value means the TS `undefined`. +// Options supplies service configuration and optional lifecycle dependencies. type Options struct { // Config is the loaded server configuration; cmd owns config.Load / Local. Config config.Config // Listener is optional prebound TCP ownership transferred on successful New. - // It lets the Local Client reserve its HTTP port before opening a tunnel. + // It lets the Local App reserve its HTTP port before opening a tunnel. Listener *net.TCPListener - // Assets is the built Browser UI. A nil FS serves the API only, which is - // the TS frontend mode "none"; webassets.FS() returns nil the same way - // when no build was embedded. + // Assets is the built Browser UI. A nil FS serves the API only. Assets fs.FS // Now returns Unix milliseconds; nil uses the wall clock. Now func() int64 @@ -75,7 +73,7 @@ type Options struct { RoomStore *room.Store } -// Server is PiikServer. +// Server owns one shared HTTP, room-authority and media service lifecycle. type Server struct { config config.Config store *room.Store @@ -108,9 +106,7 @@ type Server struct { shutdownErr error } -// New is createPiikServer minus the listening: it builds the SFU fallback, -// the room store, the site-access issuer, the frontend handler and the HTTP -// server, none of which performs I/O. +// New composes the service without binding sockets or opening the room database. func New(options Options) (*Server, error) { configuration := options.Config now := options.Now @@ -156,9 +152,7 @@ func New(options Options) (*Server, error) { EndpointMediaCopyCapacity: configuration.EndpointMediaCopyCapacity, SfuOnly: configuration.SFU != nil && configuration.SFU.Only, Ice: config.IceConfig(configuration), - // The TS signaling server read natPredictionEnabled off the same - // `ice` options object it forwarded; the Go IceConfig is already the - // wire shape, so the capability travels beside it. + // The prediction capability is separate from the wire-shaped ICE settings. NATPredictionEnabled: configuration.NATPredictionEnabled, AllowedOrigins: configuration.AllowedOrigins, SiteAccessAtUpgrade: func(request *http.Request) bool { @@ -180,13 +174,13 @@ func New(options Options) (*Server, error) { ReadHeaderTimeout: headersTimeout, ReadTimeout: requestTimeout, IdleTimeout: keepAliveTimeout, - // No WriteTimeout: Node had none, and it would cut long downloads. + // No WriteTimeout: a fixed deadline would cut long asset downloads. } return server, nil } -// newRoomStore builds the room store createPiikServer would otherwise take -// from Options.RoomStore. An empty RoomDatabasePath is the memory-only mode. +// newRoomStore uses the injected store or creates one from configuration. +// An empty RoomDatabasePath selects memory-only authority. func newRoomStore(options Options) (*room.Store, error) { if options.RoomStore != nil { return options.RoomStore, nil @@ -332,9 +326,8 @@ func (s *Server) reconcile() error { if err := s.store.Initialize(); err != nil { return err } - // Holding mu across the check and the two stores is what made the TS - // single thread safe here: a shutdown that already set closing must never - // see traffic re-enabled behind it. + // Serialize publication with shutdown; once closing is set, startup must + // never re-enable traffic. s.mu.Lock() defer s.mu.Unlock() if s.closing { @@ -355,14 +348,13 @@ func (s *Server) Close(ctx context.Context) error { return s.shutdown(ctx, false) } -// End is Close preceded by ending every room. The Client uses it, because its +// End is Close preceded by ending every room. The Local App uses it, because its // authority dies with the process. func (s *Server) End(ctx context.Context) error { return s.shutdown(ctx, true) } -// shutdown is memoised exactly as the TS shutdownOperation was: the second -// caller joins the first and gets its result, whichever entry point it used. +// Every shutdown caller joins the first operation and receives the same result. func (s *Server) shutdown(ctx context.Context, endRooms bool) error { s.shutdownOnce.Do(func() { s.shutdownErr = s.runShutdown(ctx, endRooms) }) return s.shutdownErr @@ -398,9 +390,8 @@ func (s *Server) runShutdown(ctx context.Context, endRooms bool) error { return nil } -// endAllRooms is the TS `signaling ? signaling.endAllRooms() : -// roomStore.abandonAllRooms()`: without a signaling server nobody is connected, -// so the store can be closed down directly. +// Without signaling no participant is connected, so room authority can be +// retired directly; otherwise signaling owns participant and media teardown. func (s *Server) endAllRooms(signaling *signal.Server) error { if signaling != nil { return signaling.EndAllRooms() diff --git a/internal/server/app/json.go b/internal/server/app/json.go index c5affdc74..5226ba0c1 100644 --- a/internal/server/app/json.go +++ b/internal/server/app/json.go @@ -35,7 +35,7 @@ type siteAccessBody struct { func sendJSON(writer http.ResponseWriter, status int, body any) { var buffer bytes.Buffer encoder := json.NewEncoder(&buffer) - // JSON.stringify does not escape HTML either. + // Responses have a JSON content type and are never embedded into HTML. encoder.SetEscapeHTML(false) if err := encoder.Encode(body); err != nil { panic(err) @@ -75,21 +75,13 @@ func hasRequestBody(request *http.Request) bool { return len(request.TransferEncoding) > 0 || request.ContentLength != 0 } -// requestPath is the WHATWG `url.pathname` app.ts routed and logged on. It is -// the escaped path on purpose: `new URL()` does not percent-decode, so -// "/api%2frooms" must not reach the /api/rooms route. -// -// Deviation: `new URL()` also resolves "." and ".." segments, which net/http -// leaves in the path. "/x/../api/rooms" therefore reaches the frontend handler -// here where TypeScript routed it to room creation. Only stricter: a raw path -// that already names a route still matches, so nothing can slip past an origin, -// cookie or token check that way. +// requestPath matches API routes without decoding separators or normalizing +// dot segments: "/api%2frooms" and "/x/../api/rooms" cannot alias /api/rooms. func requestPath(request *http.Request) string { return request.URL.EscapedPath() } -// cookieHeader rebuilds the single Cookie header value the Node parser handed -// to access-session.ts; Go keeps repeated Cookie headers apart. +// cookieHeader joins repeated Cookie headers before the shared cookie reader. func cookieHeader(request *http.Request) string { return strings.Join(request.Header.Values("Cookie"), "; ") } @@ -104,10 +96,8 @@ func bearerToken(request *http.Request) string { return strings.TrimPrefix(authorization, "Bearer ") } -// isUpgradeRequest reproduces the condition under which Node emitted "upgrade" -// instead of "request": the parser needs both a Connection: upgrade token and -// an Upgrade header. The SignalingServer, not this router, owns the rejection -// ladder for a path or query that is not exactly /signal. +// Upgrades require both a Connection: upgrade token and an Upgrade header. +// Signaling owns validation of the exact /signal path, query and admission. func isUpgradeRequest(request *http.Request) bool { if request.Header.Get("Upgrade") == "" { return false @@ -120,8 +110,8 @@ func isUpgradeRequest(request *http.Request) bool { return false } -// responseRecorder tracks response.headersSent so the recover wrapper can pick -// between the JSON 500 and destroying the connection, as app.ts did. +// responseRecorder lets panic recovery send JSON only before a response starts; +// a partial response must instead abort the connection. type responseRecorder struct { http.ResponseWriter wrote bool diff --git a/internal/server/app/routes.go b/internal/server/app/routes.go index 278f51820..7a69e23fb 100644 --- a/internal/server/app/routes.go +++ b/internal/server/app/routes.go @@ -48,8 +48,7 @@ func (s *Server) ServeHTTP(writer http.ResponseWriter, request *http.Request) { } signaling := s.signaling.Load() if isUpgradeRequest(request) { - // Node routed every upgrade to the SignalingServer's "upgrade" - // listener, which was attached only while traffic was accepted. + // Signaling owns every upgrade, including rejection of invalid paths. signaling.ServeHTTP(recorder, request) return } @@ -122,8 +121,7 @@ func (s *Server) route( notFoundJSON(writer, request) } -// notFoundJSON is the fallthrough app.ts handed to sirv as its next() -// callback and used when no frontend is composed at all. +// notFoundJSON handles missing assets and requests without a composed frontend. func notFoundJSON(writer http.ResponseWriter, _ *http.Request) { sendJSON(writer, http.StatusNotFound, errorBody{"Not found"}) } @@ -145,7 +143,7 @@ func allowMethod(writer http.ResponseWriter, request *http.Request, allowed stri return false } -// handleSiteAccess is handleSiteAccessRequest of app.ts. +// handleSiteAccess reports or grants site access without claiming room authority. func (s *Server) handleSiteAccess(writer http.ResponseWriter, request *http.Request) { noStoreJSON(writer) access := s.siteAccessForRequest(request) @@ -156,8 +154,7 @@ func (s *Server) handleSiteAccess(writer http.ResponseWriter, request *http.Requ Authenticated: access.isAuthenticated(cookieHeader(request)), } if status.Required && status.Authenticated { - // The Set-Cookie value is written verbatim, in the TS attribute - // order. + // Authenticated visits renew the site's idle access lifetime. writer.Header().Set("Set-Cookie", access.createCookie()) } sendJSON(writer, http.StatusOK, status) @@ -312,13 +309,8 @@ func (s *Server) handleRoomCreation( sendJSON(writer, http.StatusBadRequest, errorBody{"Invalid room request"}) return } - // TS called roomStore.createRoom directly; the store is now guarded by the - // signaling server's global mutex, so creation goes through it. There is - // no `signaling == nil` answer here, because app.ts had no getSignaling() - // check on this route: reconcile stores the signaling server before it - // sets acceptingTraffic and never clears it again, so reaching a handler - // at all means the pointer is set (the two sibling room routes keep their - // 503 only because the TypeScript had one). + // Signaling owns room mutations. Startup publishes it before accepting + // traffic, and shutdown retains the pointer for requests already admitted. created, err := signaling.CreateRoom( parsed.CodeEntryPolicy, parsed.RoomPassword.Value, @@ -335,8 +327,8 @@ func (s *Server) handleRoomCreation( sendJSON(writer, http.StatusCreated, s.createRoomResponse(created)) } -// authorizedRoomHostToken of app.ts: origin, then site access, then the bearer -// Host token, each with its own terminal response. +// authorizedRoomHostToken checks origin, site access and the bearer Host token +// in that order, each with its own terminal response. func (s *Server) authorizedRoomHostToken( writer http.ResponseWriter, request *http.Request, @@ -424,8 +416,7 @@ func roomErrorCode(err error) room.ErrorCode { return "" } -// replacementFailure maps replaceRoom errors; an unmapped error rethrows into -// the 500 handler, as the TS `throw error` did. +// replacementFailure maps room replacement errors; unmapped errors reach the 500 handler. func replacementFailure(err error) (int, string, bool) { switch roomErrorCode(err) { case room.CodeRoomLimit: @@ -439,7 +430,7 @@ func replacementFailure(err error) (int, string, bool) { } // accessFailure maps updateRoomAccess errors. Unlike the other two tables, -// every remaining RoomStoreError is a 404; only a non-store error rethrows. +// every remaining RoomStoreError is a 404; non-store errors reach the 500 handler. func accessFailure(err error) (int, string, bool) { switch code := roomErrorCode(err); code { case "": diff --git a/internal/server/app/static.go b/internal/server/app/static.go index 9c17f1171..24aaa83b3 100644 --- a/internal/server/app/static.go +++ b/internal/server/app/static.go @@ -16,7 +16,7 @@ import ( // depend on the machine; with X-Content-Type-Options: nosniff a wrong type on a // font or a webmanifest is user visible. var staticContentTypes = map[string]string{ - ".html": "text/html;charset=utf-8", // sirv appends ";charset=utf-8" with no space + ".html": "text/html;charset=utf-8", ".htm": "text/html;charset=utf-8", ".js": "text/javascript", ".mjs": "text/javascript", @@ -39,7 +39,7 @@ var staticContentTypes = map[string]string{ ".webmanifest": "application/manifest+json", ".bin": "application/octet-stream", ".xml": "text/xml", - ".ico": "", // mrmime has no .ico, so sirv sends an EMPTY Content-Type + ".ico": "", // Preserve the published empty type without sniffing. } // staticHandler serves embedded assets and falls back to index.html for SPA @@ -78,13 +78,11 @@ func staticHandler(assets fs.FS, notFound http.Handler) http.Handler { return true } - // sirv strips a trailing "/" before the lookup, and its FILES map holds - // files only, so "/assets/" can never hit a file. + // A trailing slash cannot name an asset file. if !strings.HasSuffix(request.URL.Path, "/") && serve(name) { return } - // opts.single: fall back to index.html unless the request matches one - // of sirv's two default ignores. + // Route-like paths use the SPA; files and .well-known requests do not. if !looksLikeFile(cleaned) && !strings.Contains(cleaned, "/.well-known") { if serve("index.html") { return @@ -94,9 +92,7 @@ func staticHandler(assets fs.FS, notFound http.Handler) http.Handler { }) } -// hasDotSegment reports whether any path segment starts with '.', reproducing -// totalist's filter `!opts.dotfiles && /(^\.|[\\+|\/+]\.)/.test(name)` together -// with the `/\.well-known[\\+\/]/` exemption that runs before it. +// hasDotSegment excludes hidden assets, with the published .well-known exemption. func hasDotSegment(name string) bool { if strings.Contains(name, ".well-known/") { return false @@ -109,13 +105,9 @@ func hasDotSegment(name string) bool { return false } -// looksLikeFile ports sirv's default "any extn" ignore, -// /[/]([A-Za-z\s\d~$._-]+\.\w+){1,}$/. Because the character class excludes -// "/" and the match is anchored at the end, only the last path segment can -// match: it must contain a dot with a non-empty [A-Za-z0-9_] extension after -// it, and everything before that dot must be in the class. The class is -// ASCII-only, so "/文件.js" and "/café.css" are NOT ignored and do get the -// index.html fallback. +// looksLikeFile preserves the published SPA fallback boundary: the last path +// segment needs an ASCII-style basename and a nonempty alphanumeric/underscore +// extension. Names such as "/文件.js" and "/café.css" still reach the SPA. func looksLikeFile(pathname string) bool { slash := strings.LastIndexByte(pathname, '/') if slash < 0 { From 16d6b397a7dcbceb5d0f4e02a48456cb6dd687c0 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 12:19:16 +0800 Subject: [PATCH 031/262] docs(nat): clarify outstanding Magicsock integration gates --- docs/research/nat-traversal.md | 20 ++++++++++++++++++++ docs/todo.md | 10 +++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 0a8555cd7..d80fe1116 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -406,6 +406,26 @@ The related [Pion feedback-MTU report](https://github.com/pion/interceptor/issue remains open as of 2026-10-07. Its out-of-order packet evidence does not establish the cause of any Piik field report; the pinned recorder has no byte-size option. +A 2026-10-08 reordering control flushed twelve 100-arrival reports, then supplied +one delayed packet. The recorder included already reported history and generated +a 1248-byte encrypted report from that one new arrival. Shortening the feedback +interval alone cannot bound its size. Windows and three Linux race repetitions +passed the reproduction. The latest tagged +[interceptor v0.1.49 recorder](https://github.com/pion/interceptor/blob/v0.1.49/pkg/twcc/twcc.go) +is byte-identical to the pinned recorder; Tailcat's +[October 7 revision](https://github.com/tailscale/tailcat/blob/52fbad323e9d352f1febb623e0975368e9a8cbdd/tailcat.go#L574) retains +the same payload limit and Tailscale dependency. A dependency bump does not +supply the missing per-report and compound-batch size boundary. + +The same Linux kernel lab also compared established Pion and Tailcat SRTP under +delay/jitter, 5% packet loss, reordering and a 350 ms complete interruption. +DERP was closed before these phases. Both retained their session and resumed +fresh media after restoration in three race-detector repetitions. Fresh payload +tags and unique sequence counts exclude queued or duplicate-packet success. +The original NAT and rebinding matrix repeated its prior outcomes. These controls +add bounded weak-network recovery evidence, not decoded quality or a demonstrated +direct-reachability advantage. + Kernel gateway rebinding retained full WebRTC sessions. With DERP available, fresh media resumed in about 6.4–6.5 seconds and ICE reported `connected` in about 7.1–8.1 seconds, without new SDP. With DERP closed, neither gateway direction diff --git a/docs/todo.md b/docs/todo.md index 5752d56ff..2c8dbfe90 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -1,6 +1,6 @@ # Current TODO Ledger -Last reviewed: 2026-10-07 +Last reviewed: 2026-10-08 The current 2.0 preparation includes independent investigation across this ledger. Explicit owner holds remain in force; an investigation does not itself @@ -61,8 +61,12 @@ Implementation needs a concrete useful result within the product model. have not established improved direct reachability. Seek a repeatable benefit before accepting peer-key authorization/revocation, underlay readiness and relay operation. The isolated feedback check also requires a supported RTCP - size boundary for the tunnel's MTU. Keep the experiment separate from product - routing; the P2P evidence work below owns the network/interop acceptance boundary. + size boundary for the tunnel's MTU, including reordered-history reports and + compound batches; shorter feedback intervals and the checked upstream updates + do not supply it. Kernel NAT, dual-stack, mapping replacement and bounded + loss/reordering checks exist, but do not establish that benefit. Keep the + experiment separate from product routing; the P2P evidence work below owns + the network/interop acceptance boundary. - [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) identifies UDP/storage requirements but establishes no cost saving. Keep the existing P2P-only Demo until a provider and operating boundary are accepted. From d918bb53eb5a3114fba995f6baf6401a459c83d1 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 12:54:46 +0800 Subject: [PATCH 032/262] fix(presentation): align docs branding and clarify SFU diagnostics --- docs/research/nat-traversal.md | 12 ++++++++++++ docs/todo.md | 7 ++++++- internal/server/signal/router.go | 3 ++- internal/server/signal/router_sfu_signal_test.go | 3 +++ site/assets/brand.css | 6 +++--- site/assets/brand.js | 6 +++++- site/docs/.vitepress/config.mjs | 1 + site/docs/.vitepress/theme/index.js | 6 +++--- site/docs/.vitepress/theme/style.css | 3 +-- 9 files changed, 36 insertions(+), 11 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index d80fe1116..9ab773b60 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -417,6 +417,18 @@ is byte-identical to the pinned recorder; Tailcat's the same payload limit and Tailscale dependency. A dependency bump does not supply the missing per-report and compound-batch size boundary. +A local upstream-layer repair candidate now bounds TWCC reports through the +existing recorder split loop and sends them separately before encryption. Its +fixed 1200-byte plaintext budget is experimental, not a supported upstream +option. Unit checks preserve loss, reordered history, wrapped sequence numbers +and arrival times; restoring either old recorder or writer fails its regression +check. A real interceptor-to-Tailcat control delivered all 1500 statuses in two +reports after DERP closed, with a largest encrypted datagram of 1212 bytes. +Windows and three Linux race repetitions passed, as did the upstream TWCC suite +with the candidate. The isolated probe retains the opt-in patch and reproduction; +production dependencies are unchanged. This narrows one integration obstacle, +not the missing direct-reachability benefit or other feedback producers' bounds. + The same Linux kernel lab also compared established Pion and Tailcat SRTP under delay/jitter, 5% packet loss, reordering and a 350 ms complete interruption. DERP was closed before these phases. Both retained their session and resumed diff --git a/docs/todo.md b/docs/todo.md index 2c8dbfe90..d170af38d 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -63,7 +63,9 @@ Implementation needs a concrete useful result within the product model. relay operation. The isolated feedback check also requires a supported RTCP size boundary for the tunnel's MTU, including reordered-history reports and compound batches; shorter feedback intervals and the checked upstream updates - do not supply it. Kernel NAT, dual-stack, mapping replacement and bounded + do not supply it. An isolated upstream repair candidate passes actual encrypted + feedback and race checks; it is not a supported or adopted dependency change. + Kernel NAT, dual-stack, mapping replacement and bounded loss/reordering checks exist, but do not establish that benefit. Keep the experiment separate from product routing; the P2P evidence work below owns the network/interop acceptance boundary. @@ -117,6 +119,9 @@ of a reporter's cause; research owns the completed experiments and their limits. [#453](https://github.com/TNTcraftHIM/Piik/issues/453) has no matched endpoint reports to distinguish network reachability from a product failure. The v1.6.3 candidate-timeout report lacks share-start and SDP/ICE evidence. + A self-hosted SFU report supplies only a screenshot of a third-party diagnosis; + obtain the original Viewer decode/`route-ready` events and matching Server + revision before attributing timeout to the committed-publication flag. - [ ] **Camera and Host microphone device coverage.** Check real audio levels, echo, device replacement/native mixing and phone camera permission, orientation and background behavior. Include OBS virtual-camera/audio-input sound quality diff --git a/internal/server/signal/router.go b/internal/server/signal/router.go index 701a4ebed..a0951e2b8 100644 --- a/internal/server/signal/router.go +++ b/internal/server/signal/router.go @@ -714,6 +714,7 @@ func (r *router) handleRouteReady(participant authenticatedRouteParticipant, mes "participant", r.debugPeer(participant.roomID, participant.peerID), "revision", revision, "accepted", settled.Accepted, + "committed", settled.Committed, "exhausted", len(settled.FailedPeerIDs) > 0) r.releaseResources(settled.Released) if settled.Committed && settled.Accepted { @@ -1189,7 +1190,7 @@ func (r *router) broadcastActive(roomID string, rm *roomRuntime) { r.debug(roomID, "active-topology", "revision", snapshot.Revision, "paused", snapshot.Paused, - "hostPublication", snapshot.HostPublication != nil && snapshot.HostPublication.PhysicalActive, + "committedHostPublication", snapshot.HostPublication != nil && snapshot.HostPublication.PhysicalActive, "routes", routes) } assignments := r.assignments(roomID, snapshot, nil, "", "") diff --git a/internal/server/signal/router_sfu_signal_test.go b/internal/server/signal/router_sfu_signal_test.go index cfa4f6130..7b0487b97 100644 --- a/internal/server/signal/router_sfu_signal_test.go +++ b/internal/server/signal/router_sfu_signal_test.go @@ -221,6 +221,9 @@ func TestEmbeddedSfuAuthenticatedWebSocketCarriesActualRTP(t *testing.T) { if room.controller.Operation() == nil { t.Fatal("transport/RTP arrival committed without client decoded-frame proof") } + if room.controller.Publication() != nil { + t.Fatal("preparing media was advertised as a committed Host publication") + } }) viewer.sendJSON(map[string]any{"type": "route-ready", "revision": viewerConfig.Revision, "phase": "prepare"}) if !waitFor(t, time.Second, func() bool { diff --git a/site/assets/brand.css b/site/assets/brand.css index 15cc66772..4f078e784 100644 --- a/site/assets/brand.css +++ b/site/assets/brand.css @@ -1,8 +1,8 @@ -.identity { display:flex; gap:11px; align-items:center; text-decoration:none; font-size:32px; font-weight:800; letter-spacing:-.065em; line-height:1; color:inherit; } +.identity { display:flex; gap:11px; align-items:baseline; text-decoration:none; font-size:32px; font-weight:800; letter-spacing:-.065em; line-height:1; color:inherit; } .identity .mascot { width:37px; height:37px; overflow:visible; } /* Align the feet with the wordmark baseline, allowing for the SVG's bottom inset. */ -.identity[data-brand],.footer-identity { align-items:baseline; } -.identity[data-brand] .mascot,.footer-identity > svg { translate:0 .1em; } +.footer-identity { align-items:baseline; } +.identity .mascot,.footer-identity > svg { translate:0 .1em; } .identity-dot { color:#e4572e; } .identity .mascot-face,.identity .mascot-eye-open,.identity .mascot-sparkles { transform-box:fill-box; transform-origin:center; } @media (prefers-reduced-motion:no-preference) { diff --git a/site/assets/brand.js b/site/assets/brand.js index c2594636a..ee710100e 100644 --- a/site/assets/brand.js +++ b/site/assets/brand.js @@ -15,9 +15,13 @@ export function mascotMarkup(id, fill = 'none') { `; } +export function brandMarkup(id) { + return `Piik.`; +} + export function mountBrands() { document.querySelectorAll('[data-brand]').forEach((link, index) => { - link.innerHTML = `Piik.`; + link.innerHTML = brandMarkup(`brand-${index}`); const replay = () => link.getAnimations({ subtree: true }).forEach(animation => { animation.currentTime = 0; }); link.addEventListener('pointerenter', replay); link.addEventListener('focus', replay); diff --git a/site/docs/.vitepress/config.mjs b/site/docs/.vitepress/config.mjs index d889cc3c7..67d30a3e2 100644 --- a/site/docs/.vitepress/config.mjs +++ b/site/docs/.vitepress/config.mjs @@ -48,6 +48,7 @@ export default defineConfig({ }, }, themeConfig: { + siteTitle: false, nav: [{ text: 'Website', link: 'https://piik.tv/?lang=en' }, { text: 'Use online', link: 'https://demo.piik.tv' }], socialLinks: [{ icon: 'github', link: repository }], outline: [2, 3], diff --git a/site/docs/.vitepress/theme/index.js b/site/docs/.vitepress/theme/index.js index 5035c263a..0f8681128 100644 --- a/site/docs/.vitepress/theme/index.js +++ b/site/docs/.vitepress/theme/index.js @@ -1,7 +1,7 @@ import DefaultTheme from 'vitepress/theme-without-fonts'; import { useData, withBase } from 'vitepress'; import { h } from 'vue'; -import { mascotMarkup } from '../../../assets/brand.js'; +import { brandMarkup } from '../../../assets/brand.js'; import { initialLanguage, rememberLanguage } from '../../../assets/language.js'; import '../../../assets/brand.css'; import './style.css'; @@ -11,8 +11,8 @@ export default { Layout() { const { lang } = useData(); return h(DefaultTheme.Layout, null, { - 'nav-bar-title-before': () => h('span', { class: 'identity', 'aria-hidden': 'true', - innerHTML: `${mascotMarkup('docs-brand')}` }), + 'nav-bar-title-before': () => h('span', { class: 'identity', + innerHTML: brandMarkup('docs-brand') }), 'doc-bottom': () => h('p', { class: 'docs-license' }, [ h('a', { href: withBase('/third-party-licenses.txt') }, lang.value === 'zh-CN' ? '第三方软件许可' : 'Third-party licenses'), diff --git a/site/docs/.vitepress/theme/style.css b/site/docs/.vitepress/theme/style.css index 978131391..a68df7e05 100644 --- a/site/docs/.vitepress/theme/style.css +++ b/site/docs/.vitepress/theme/style.css @@ -26,8 +26,7 @@ --vp-c-divider: #3b5155; --vp-code-block-bg: #14232b; } -.VPNavBarTitle .title { gap: 8px; font-size: 24px; font-weight: 800; letter-spacing: -.045em; } -.VPNavBarTitle .title::after { content: "."; margin-left: -8px; color: #e4572e; } +.VPNavBarTitle .identity { gap: 8px; font-size: 24px; letter-spacing: -.045em; } .VPNavBarTitle .mascot { width: 30px; height: 30px; } .vp-doc h1 { letter-spacing: -.035em; } .vp-doc p, .vp-doc li { line-height: 1.85; } From ebff6382312468038a203875cffbd1a480664d49 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:10:56 +0800 Subject: [PATCH 033/262] docs(media): record feedback and capture-border evidence limits --- docs/research/nat-traversal.md | 10 +++++++++- docs/research/native-client-lifecycle.md | 8 ++++++++ docs/todo.md | 15 ++++++++------- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 9ab773b60..5c59e0dae 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -427,7 +427,15 @@ reports after DERP closed, with a largest encrypted datagram of 1212 bytes. Windows and three Linux race repetitions passed, as did the upstream TWCC suite with the candidate. The isolated probe retains the opt-in patch and reproduction; production dependencies are unchanged. This narrows one integration obstacle, -not the missing direct-reachability benefit or other feedback producers' bounds. +not the missing direct-reachability benefit or a supported upstream MTU contract. + +The current Go feedback paths were also traced end to end. Pion's default NACK +generator retains a 512-packet window and writes one report per media SSRC; +a full-loss/wrapped-gap control with two SSRCs stays within 156 encrypted bytes. +Normal sender/receiver reports are written separately per stream. Piik's SFU +ingress callback forwards only PLI/FIR, and LiveKit's reachable XR response contains one +DLRR block. No additional oversized producer was found in these paths; this +does not bound arbitrary caller-created RTCP batches or future configurations. The same Linux kernel lab also compared established Pion and Tailcat SRTP under delay/jitter, 5% packet loss, reordering and a 350 ms complete interruption. diff --git a/docs/research/native-client-lifecycle.md b/docs/research/native-client-lifecycle.md index 291a7adfb..b6e2e30a5 100644 --- a/docs/research/native-client-lifecycle.md +++ b/docs/research/native-client-lifecycle.md @@ -343,6 +343,14 @@ This establishes a retirement defect and its local repair, not the cause of every reported Windows 11 border: denied consent and other active captures remain distinct Windows-owned limits. +The source-preview call chain has a separate, finite lifetime. Closing the +picker invalidates queued thumbnails and their results; it does not kill the +in-flight helper. That helper closes its WGC session before writing the bitmap. +Control-session loss or the three-second preview deadline can still terminate +it through `CommandContext`. The 2026-10-08 review did not reproduce a retained +border from this path. Its desktop-pixel control could not see the controlled +source, so capture/permission logs alone were not accepted as visual evidence. + The upstream [display-resolution change report](https://github.com/robmikh/Win32CaptureSample/issues/59) is still open as of 2026-10-07: it reports borderless display capture losing that preference after a mode change. It gives neither a current Windows build nor a diff --git a/docs/todo.md b/docs/todo.md index d170af38d..30c175b4a 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -15,13 +15,6 @@ Git/PRs own completed history. coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. -- [ ] **Windows 11 capture border remains visible.** Reopened by a new report - on 2026-10-07. Recheck App/Browser capture ownership, border permission/API - results, live setting changes, display-mode changes, concurrent captures and - source-thumbnail cancellation/retirement before attributing this to Windows. - Obtain the affected version/path/build when available; the report - does not yet identify them. [Capture evidence](./research/native-client-lifecycle.md#windows-capture-borders) - owns the prior graceful-retirement and concurrent-capture checks. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance are complete. Integrate through the next owner-approved publishing merge and verify the separate workflow with its original CI artifacts. Mirror failure @@ -88,6 +81,14 @@ Implementation needs a concrete useful result within the product model. These reports remain unmatched. A local repair or successful test is not proof of a reporter's cause; research owns the completed experiments and their limits. +- [ ] **Windows 11 capture border remains visible.** The new report does not + identify the affected version, capture path or Windows build. The reviewed + picker/start/source-change paths preserve the border preference; no additional + confirmed defect was found. [Capture evidence](./research/native-client-lifecycle.md#windows-capture-borders) + owns the permission correction, retirement checks and preview-cancellation + boundary. Visible permission changes, display-mode changes and the reporter's + setup still need matched physical evidence; API success does not prove that + Windows hid the border. - [ ] **macOS and Linux physical capture.** Complete the remaining capture/audio checks in [verification status](./verification-status.md#device-evidence-boundary), including an Intel Mac with a usable hardware H.264 encoder, an ARM64 Linux From d9251c6364e0dd6cac5daa4ade449025c072c3f4 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:48:24 +0800 Subject: [PATCH 034/262] docs(nat): record Magicsock authorization blockers --- docs/research/nat-traversal.md | 47 +++++++++++++++++++++++++++------- docs/todo.md | 6 ++++- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 5c59e0dae..05c6372af 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -320,14 +320,14 @@ isolated branch; production dependencies and routing remain unchanged. ### Integration Cost And Recovery Controls The isolated `spike/magicsock-assessment` branch retains the comparison probes. -These nested Go modules are outside Piik's -build and dependency graph. Repeated local Tailcat bootstrap, peer denial, -revocation and synthetic SRTP media checks passed; the raw magicsock direct-path -matrix again found the same reachable classes as fresh Pion ICE. Retired shared -mapping-cache reproductions remain causal controls, not the current baseline. +The nested modules are outside Piik's build. Local bootstrap, peer denial and +synthetic SRTP media checks passed. Initial withdrawal checks observed stopped +outbound media; the stronger authorization checks below found revocation defects. +The raw magicsock direct-path matrix found the same reachable classes as fresh +Pion ICE. Retired mapping-cache reproductions remain controls, not the baseline. [Tailcat](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/README.md) -provides a supported Go reuse lead: magicsock plus WireGuard, with DERP bootstrap +provides an upstream Go reuse candidate: magicsock plus WireGuard, with DERP bootstrap and relay fallback. Its Browser path at that pin is relay-only. In the isolated Linux kernel NAT comparison on 2026-10-05, both stacks connected directly through ordinary/ordinary and open/hard gateways in either direction. Neither delivered @@ -365,9 +365,10 @@ adoption boundary. The 2026-10-07 experiment through `f0ca8056` keeps Pion's ICE/DTLS/SRTP over Tailcat. Each physical connection owns a virtual UDP tuple and mux; its one-shot listener retires after admission. Connections to one authorized peer may share -the tunnel. Closing or aborting one flow preserves healthy siblings; revoking -the fixture peer key denies admission and disconnects that tunnel. These checks -do not implement Piik room grants or endpoint exchange. +the tunnel. Closing or aborting one flow preserves healthy siblings. Initial +revocation checks stopped outbound media and denied a fresh client; they did not +prove that the old client's inbound traffic was rejected. These checks do not +implement Piik room grants or endpoint exchange. Repeated lifecycle and Linux race checks cover overlapping media preparation, rollback, replacement, canceled preparation followed by retry, bounded datagrams @@ -468,6 +469,34 @@ ignored. Further product work needs a measurable network benefit and accepted room authorization, resource and relay-operation owners, not more loopback successes. +### Peer Authorization Boundary + +The 2026-10-08 three-peer control found that Tailcat forgets a revoked key but +leaves its WireGuard session alive: fresh inbound UDP still arrives after +`DisconnectClient` returns and `PeerKey` no longer recognizes it. A separate +controlled race lets a late positive `AllowClient` result re-admit a revoked +key. Both defects were independently reproduced; Tailcat's checked +[October 7 source](https://github.com/tailscale/tailcat/blob/52fbad323e9d352f1febb623e0975368e9a8cbdd/tailcat.go) +retains both mechanisms. Piik's current transport does not use this dependency. + +An opt-in patch syncs the existing engine after releasing the backend lock and +invalidates late results in the existing pending-hook map. Three Windows and +Linux race repetitions reject both traffic directions and new admission, +preserve two healthy peers' media, and permit a later explicit grant. Removing +either correction fails its regression; upstream admission/disconnect checks +pass. This is a partial repair, not safe-to-adopt authorization. + +A deeper unit reproduction pauses WireGuard's lazy peer-config lookup, withdraws +authority and calls `RemovePeer`, then releases the old result. `LookupPeer` +still installs an active peer. Both the pinned `0a83b87f7908` and +[October 2 revision](https://github.com/tailscale/wireguard-go/blob/18e99eba5393b8c9ea513ab7a1db1a12625422ca/device/device.go) +reproduce it, including three Windows and Linux race repetitions on the newer +version. This proves an authority/install race, not an end-to-end Piik exploit. +A supported atomic retirement/creation boundary remains necessary below Tailcat; +its partial patch cannot close that race. The probe retains the patch and failing +reproduction. Piik deny lists or repeated resets are not substitutes. This, +feedback sizing and unproven direct-reachability gains block product adoption. + ## Native Mapping Cache And Retry Lifetimes Pion ICE `v4.4.0` caches a shared socket's STUN mapping by destination for 25 diff --git a/docs/todo.md b/docs/todo.md index 30c175b4a..af53972e4 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -53,7 +53,11 @@ Implementation needs a concrete useful result within the product model. [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) have not established improved direct reachability. Seek a repeatable benefit before accepting peer-key authorization/revocation, underlay readiness and - relay operation. The isolated feedback check also requires a supported RTCP + relay operation. The [authorization assessment](./research/nat-traversal.md#peer-authorization-boundary) + reproduces stale WireGuard sessions, late admission and a deeper lazy-peer + creation race. Two isolated Tailcat corrections pass bounded checks; the + lower-level race remains unresolved, so product adoption stays blocked. + The isolated feedback check also requires a supported RTCP size boundary for the tunnel's MTU, including reordered-history reports and compound batches; shorter feedback intervals and the checked upstream updates do not supply it. An isolated upstream repair candidate passes actual encrypted From 08e97d7721328307236c580a0055139803da42ed Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:09:10 +0800 Subject: [PATCH 035/262] docs(nat): record combined transport repair evidence --- docs/research/nat-traversal.md | 49 +++++++++++++++++----------------- docs/todo.md | 5 ++-- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 05c6372af..a3ecfbd6d 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -304,18 +304,13 @@ hold endpoint pairs, NAT/filtering, IPv6, network transitions and relay policy constant, measuring direct success, relay use and first media separately. No new transport or universal success-rate claim is accepted by this review. -The isolated `spike/magicsock-feasibility` Go experiment on 2026-10-04 carried -Pion ICE/DTLS and matching synthetic audio/video RTP payloads over magicsock, -then delivered new payloads after both UDP endpoints changed without new SDP. -Loopback and separate address-and-port-filtering NATs passed, including five -race-detector repetitions. The fixture supplied authenticated peer identities -and endpoint updates; it did not test automatic OS network-change detection, -decoded playback or real routers. A separate seven-pair NAT matrix found the -same direct reachability as Pion within four-second windows, not a field success -rate. Go integration is feasible; no Rust requirement was found. Product use -still needs room-authorized key/endpoint exchange, MTU/resource limits and mixed -Browser/Native behavior. The nested probe module and full evidence stay on that -isolated branch; production dependencies and routing remain unchanged. +The initial Go control carried Pion ICE/DTLS/SRTP over magicsock and delivered +fresh media after both UDP endpoints changed without new SDP. Loopback and +separate address-and-port-filtering NATs passed five race repetitions. The +fixture supplied peer identities and endpoint updates; it did not prove automatic +OS network-change detection, decoded playback or real-router behavior. Its +seven-pair direct matrix matched Pion within four-second windows. Go reuse is +feasible; product integration and network benefit require the controls below. ### Integration Cost And Recovery Controls @@ -479,23 +474,29 @@ key. Both defects were independently reproduced; Tailcat's checked [October 7 source](https://github.com/tailscale/tailcat/blob/52fbad323e9d352f1febb623e0975368e9a8cbdd/tailcat.go) retains both mechanisms. Piik's current transport does not use this dependency. -An opt-in patch syncs the existing engine after releasing the backend lock and -invalidates late results in the existing pending-hook map. Three Windows and -Linux race repetitions reject both traffic directions and new admission, -preserve two healthy peers' media, and permit a later explicit grant. Removing -either correction fails its regression; upstream admission/disconnect checks -pass. This is a partial repair, not safe-to-adopt authorization. +An opt-in Tailcat patch syncs the existing engine after releasing the backend +lock and invalidates late results in its pending-hook map. Three Windows +repetitions and three Linux race runs reject both traffic directions and new +admission, preserve two healthy peers' media and permit a later explicit grant. +Removing either correction fails its regression; upstream admission/disconnect +checks pass. This repairs the Tailcat layer only. A deeper unit reproduction pauses WireGuard's lazy peer-config lookup, withdraws authority and calls `RemovePeer`, then releases the old result. `LookupPeer` still installs an active peer. Both the pinned `0a83b87f7908` and [October 2 revision](https://github.com/tailscale/wireguard-go/blob/18e99eba5393b8c9ea513ab7a1db1a12625422ca/device/device.go) -reproduce it, including three Windows and Linux race repetitions on the newer -version. This proves an authority/install race, not an end-to-end Piik exploit. -A supported atomic retirement/creation boundary remains necessary below Tailcat; -its partial patch cannot close that race. The probe retains the patch and failing -reproduction. Piik deny lists or repeated resets are not substitutes. This, -feedback sizing and unproven direct-reachability gains block product adoption. +reproduce it. An isolated patch on the newer version fences late installation +by the peer table's generation and starts lazy peers under that same lock. +Removal, lookup replacement, close and regrant checks pass; removing the fence +reproduces the failure. The full device suite passes on Windows and Linux +(race detection on Linux), and its lock-order analyzer passes. Combined Tailcat/ +WireGuard media and bounded TWCC checks pass three repetitions on both platforms. +The combined patches also preserve the kernel NAT, dual-stack, rebinding and +weak-network results in the Linux race run. +One removal also invalidates other pending lookups; existing media is untouched. +This coarse experimental fence needs review before adoption. The probe retains +the patches; supported dependency contracts and demonstrated network benefit +remain open. Piik's production dependencies and routing are unchanged. ## Native Mapping Cache And Retry Lifetimes diff --git a/docs/todo.md b/docs/todo.md index af53972e4..141a2a31c 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -55,8 +55,9 @@ Implementation needs a concrete useful result within the product model. before accepting peer-key authorization/revocation, underlay readiness and relay operation. The [authorization assessment](./research/nat-traversal.md#peer-authorization-boundary) reproduces stale WireGuard sessions, late admission and a deeper lazy-peer - creation race. Two isolated Tailcat corrections pass bounded checks; the - lower-level race remains unresolved, so product adoption stays blocked. + creation race. Isolated Tailcat/WireGuard corrections pass bounded checks; + supported upstream fixes and the coarse generation fence's admission impact + still need review before product adoption. The isolated feedback check also requires a supported RTCP size boundary for the tunnel's MTU, including reordered-history reports and compound batches; shorter feedback intervals and the checked upstream updates From 77e293fb0612d84551b9d28aaa90a624914333b5 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:33:48 +0800 Subject: [PATCH 036/262] docs(nat): record scoped admission isolation evidence --- docs/research/nat-traversal.md | 24 ++++++++++++------------ docs/todo.md | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index a3ecfbd6d..1e771d94b 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -485,18 +485,18 @@ A deeper unit reproduction pauses WireGuard's lazy peer-config lookup, withdraws authority and calls `RemovePeer`, then releases the old result. `LookupPeer` still installs an active peer. Both the pinned `0a83b87f7908` and [October 2 revision](https://github.com/tailscale/wireguard-go/blob/18e99eba5393b8c9ea513ab7a1db1a12625422ca/device/device.go) -reproduce it. An isolated patch on the newer version fences late installation -by the peer table's generation and starts lazy peers under that same lock. -Removal, lookup replacement, close and regrant checks pass; removing the fence -reproduces the failure. The full device suite passes on Windows and Linux -(race detection on Linux), and its lock-order analyzer passes. Combined Tailcat/ -WireGuard media and bounded TWCC checks pass three repetitions on both platforms. -The combined patches also preserve the kernel NAT, dual-stack, rebinding and -weak-network results in the Linux race run. -One removal also invalidates other pending lookups; existing media is untouched. -This coarse experimental fence needs review before adoption. The probe retains -the patches; supported dependency contracts and demonstrated network benefit -remain open. Piik's production dependencies and routing are unchanged. +reproduce it. An isolated patch on the newer version tracks in-flight config +lookups at the peer table and starts lazy peers under that same lock. Removing +a key retires only its pending calls; lookup replacement and close retire all. +Tokens are reclaimed on completion without accumulating historical keys. +An unrelated-key control rejects a global generation fence: it cancels valid +concurrent admission. Scoped tokens preserve that admission, overlapping grants +and cleanup; removing the retirement check reproduces stale installation. +The full device suite and lock-order analyzer pass, including three Linux race +runs. Combined media and TWCC checks pass on Windows and Linux; the Linux kernel +NAT, dual-stack, rebinding and weak-network controls also pass. Supported dependency +contracts and demonstrated network benefit remain open. The probe retains the +isolated patches; Piik's production transport is unchanged. ## Native Mapping Cache And Retry Lifetimes diff --git a/docs/todo.md b/docs/todo.md index 141a2a31c..7b5b00e0d 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -56,8 +56,8 @@ Implementation needs a concrete useful result within the product model. relay operation. The [authorization assessment](./research/nat-traversal.md#peer-authorization-boundary) reproduces stale WireGuard sessions, late admission and a deeper lazy-peer creation race. Isolated Tailcat/WireGuard corrections pass bounded checks; - supported upstream fixes and the coarse generation fence's admission impact - still need review before product adoption. + the scoped lookup fence also preserves unrelated concurrent admission. + Supported upstream fixes still need review before product adoption. The isolated feedback check also requires a supported RTCP size boundary for the tunnel's MTU, including reordered-history reports and compound batches; shorter feedback intervals and the checked upstream updates From 213ffa5b18d38ac022eaaf19f6bbdcd12ffe3e97 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:21:40 +0800 Subject: [PATCH 037/262] docs(nat): record native capability comparisons --- docs/research/nat-traversal.md | 27 +++++++++++++++++++++++---- docs/todo.md | 5 +++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 1e771d94b..6d2e88379 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -350,10 +350,9 @@ diagnostic Ping polling or wider product deadlines as integration shortcuts. Keep product integration conditional on a repeatable benefit on matched networks, with direct coverage, relay coverage, recovery and first picture measured -separately. Go feasibility alone does not establish that benefit. Gateway -mapping, Piik prediction, physical routers and the full mixed Piik topology -remain outside the comparison. The isolated controls below do not change that -adoption boundary. +separately. Go feasibility alone does not establish that benefit. The actual +Native-engine controls below cover gateway mapping, prediction and IPv6; +physical routers and the full mixed Piik topology remain outside the comparison. ### Isolated Connection-Layer Acceptance @@ -459,6 +458,26 @@ direct address did not establish that both directions were direct. IPv4 TCP still supplied bootstrap, so this is not an IPv6-only or physical ISP check. These controls add no demonstrated direct-reachability advantage for Magicsock. +The actual Native `mediaedge.Engine` Host/Viewer now runs through the same kernel +gateways, including local playback and the unchanged TypeScript survey predictor. +Three Linux race repetitions received ten distinct RTP packets containing the +generated VP8 frame, excluding padding/duplicates. With no gateway service, +mapping enabled did not hold ordinary media behind discovery: delivery took +about 0.8 seconds. Either usable IP family also carried media. Sequential-port +NAT failed without prediction and delivered through a predicted candidate when +enabled; random-port NAT remained unreachable in the bounded checks. A real +MiniUPnPd gateway made a hard/hard pair reachable when mapping was enabled. +These are encoded-media checks, not capture, Browser decode or room admission. + +Tailcat's matched controls explicitly link its [optional portmapper feature](https://github.com/tailscale/tailscale/blob/a0e471a35b8f/feature/portmapper/portmapper.go); +clearing its disable flag alone does not register that optional feature. Gateway +mapping then passed three race repetitions after DERP closure. The 20-second +comparison distinguishes fresh direct SRTP from relay delivery and retains the +same gateway filtering. Without mapping service, Tailcat's random and sequential +pairs remained relay-only; no extra direct-reachability class over Piik was found. +Its relay bootstrap and Piik's local playback differ, so their startup timings +are not a first-picture benchmark. Product dependencies and routing are unchanged. + The experiment's README owns commands and fixture boundaries; raw output remains ignored. Further product work needs a measurable network benefit and accepted room authorization, resource and relay-operation owners, not more loopback diff --git a/docs/todo.md b/docs/todo.md index 7b5b00e0d..2b96fa26d 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -63,8 +63,9 @@ Implementation needs a concrete useful result within the product model. compound batches; shorter feedback intervals and the checked upstream updates do not supply it. An isolated upstream repair candidate passes actual encrypted feedback and race checks; it is not a supported or adopted dependency change. - Kernel NAT, dual-stack, mapping replacement and bounded - loss/reordering checks exist, but do not establish that benefit. Keep the + Kernel checks now include Piik's actual Native engine, shared predictor and + real gateway mapping alongside dual-stack, mapping replacement and bounded + loss/reordering. They still do not establish extra direct reachability. Keep the experiment separate from product routing; the P2P evidence work below owns the network/interop acceptance boundary. - [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) From be269532878376df6ac130353e9f3d39eacf4c60 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:59:49 +0800 Subject: [PATCH 038/262] docs(nat): qualify multi-destination traversal evidence --- docs/research/nat-traversal.md | 22 +++++++++++++++++----- docs/todo.md | 7 ++++--- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 6d2e88379..aee1453b4 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -465,7 +465,8 @@ generated VP8 frame, excluding padding/duplicates. With no gateway service, mapping enabled did not hold ordinary media behind discovery: delivery took about 0.8 seconds. Either usable IP family also carried media. Sequential-port NAT failed without prediction and delivered through a predicted candidate when -enabled; random-port NAT remained unreachable in the bounded checks. A real +enabled, with the changing-port gateway on either endpoint; random-port NAT +remained unreachable in the bounded checks. A real MiniUPnPd gateway made a hard/hard pair reachable when mapping was enabled. These are encoded-media checks, not capture, Browser decode or room admission. @@ -473,10 +474,21 @@ Tailcat's matched controls explicitly link its [optional portmapper feature](htt clearing its disable flag alone does not register that optional feature. Gateway mapping then passed three race repetitions after DERP closure. The 20-second comparison distinguishes fresh direct SRTP from relay delivery and retains the -same gateway filtering. Without mapping service, Tailcat's random and sequential -pairs remained relay-only; no extra direct-reachability class over Piik was found. -Its relay bootstrap and Piik's local playback differ, so their startup timings -are not a first-picture benchmark. Product dependencies and routing are unchanged. +same gateway filtering. Its relay bootstrap and Piik's local playback differ, +so startup timings are not a first-picture benchmark. + +A 2026-10-08 fairness control found that Tailcat's single bootstrap region leaves +`MappingVariesByDestIP` unknown even with three STUN addresses: upstream +[netcheck](https://github.com/tailscale/tailscale/blob/a0e471a35b8f/net/netcheck/netcheck.go) +cancels the region's other probes after its first reply. A lab-only map projection +gives those real addresses independent probe regions and pins the original relay; +it is not a supported Tailcat option. Twelve Linux race cases verify classification +on the actual media socket, keep media flowing until relay closure at 12 seconds, +and require a fresh payload before the 20-second bound. Random/sequential pairs +remain relay-only in both directions, with or without an absent-gateway mapping +probe; an available gateway still establishes direct media. No extra direct path +over Piik was found. Physical routers and field success rates remain unmeasured. +Product dependencies and routing are unchanged. The experiment's README owns commands and fixture boundaries; raw output remains ignored. Further product work needs a measurable network benefit and accepted diff --git a/docs/todo.md b/docs/todo.md index 2b96fa26d..dda565d39 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -63,9 +63,10 @@ Implementation needs a concrete useful result within the product model. compound batches; shorter feedback intervals and the checked upstream updates do not supply it. An isolated upstream repair candidate passes actual encrypted feedback and race checks; it is not a supported or adopted dependency change. - Kernel checks now include Piik's actual Native engine, shared predictor and - real gateway mapping alongside dual-stack, mapping replacement and bounded - loss/reordering. They still do not establish extra direct reachability. Keep the + Kernel checks now include Piik's actual Native engine, shared predictor, + real gateway mapping and verified multi-destination netcheck alongside + dual-stack, mapping replacement and bounded loss/reordering. They still do not + establish extra direct reachability. Keep the experiment separate from product routing; the P2P evidence work below owns the network/interop acceptance boundary. - [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) From 9e6f2a22cd3082655dd510e43a0bb545bfd4047d Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:23:05 +0800 Subject: [PATCH 039/262] docs(nat): record bounded recovery lifecycle evidence --- docs/research/nat-traversal.md | 35 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index aee1453b4..ab1336822 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -432,24 +432,23 @@ ingress callback forwards only PLI/FIR, and LiveKit's reachable XR response cont DLRR block. No additional oversized producer was found in these paths; this does not bound arbitrary caller-created RTCP batches or future configurations. -The same Linux kernel lab also compared established Pion and Tailcat SRTP under -delay/jitter, 5% packet loss, reordering and a 350 ms complete interruption. -DERP was closed before these phases. Both retained their session and resumed -fresh media after restoration in three race-detector repetitions. Fresh payload -tags and unique sequence counts exclude queued or duplicate-packet success. -The original NAT and rebinding matrix repeated its prior outcomes. These controls -add bounded weak-network recovery evidence, not decoded quality or a demonstrated -direct-reachability advantage. - -Kernel gateway rebinding retained full WebRTC sessions. With DERP available, -fresh media resumed in about 6.4–6.5 seconds and ICE reported `connected` in about -7.1–8.1 seconds, without new SDP. With DERP closed, neither gateway direction -recovered within 20 seconds. Fresh media and ICE state are separate observations; -Piik's disconnected grace, restart and retirement need a full integration -control before adoption. Underlay preparation must fit the existing operation -deadline, not start a second budget. Local cold setup of about 3.3 seconds for -Tailcat versus 0.2 seconds for Pion disabled STUN/gateway mapping and is not a WAN -startup comparison. +Kernel queue controls compare established Pion and Tailcat SRTP under delay, +jitter, 5% loss and reordering with DERP closed. Both retained their session and +resumed after a 350 ms blackout in three race repetitions, and after a separate +nine-second blackout. Fresh tags and unique sequences exclude queued/duplicate +success. These are bounded recovery checks, not decoded quality or field coverage. + +Kernel gateway rebinding retained full WebRTC: DERP restored fresh media in about +6.4–6.5 seconds and ICE `connected` in 7.1–8.1 seconds, without new SDP. Its recorded +disconnected interval was about two seconds, inside Piik's three-second grace; +the nine-second blackout reached about four seconds disconnected. Actual Native +Viewer-owner replays cancel grace or request one restart, clear recovery on +connection and retain the receiver. They mock App events, so do not exercise the +restart exchange over Tailcat or prove full integration. With DERP closed, neither +gateway direction recovered within 20 seconds. Fresh media, ICE state and a +unilateral direct address are separate facts. Underlay setup must fit the existing +operation deadline. Local cold setup of about 3.3 seconds for Tailcat versus 0.2 +seconds for Pion disabled STUN/gateway mapping; it is not a WAN startup comparison. Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof From 0fcf709601bb129cbdf1a81d97d59dcc89c46759 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:51:41 +0800 Subject: [PATCH 040/262] docs(media): qualify new connection reports and local fallback evidence --- docs/research/browser-local-encoding-pool.md | 16 ++++++++++++++++ docs/todo.md | 16 +++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/docs/research/browser-local-encoding-pool.md b/docs/research/browser-local-encoding-pool.md index 873c44832..a462562cd 100644 --- a/docs/research/browser-local-encoding-pool.md +++ b/docs/research/browser-local-encoding-pool.md @@ -101,6 +101,22 @@ around fallback was 271 ms for H264 and 80 ms for VP8. Source-wide fallback therefore has a measurable handoff cost, even without a resolution drop or connection failure. This does not establish low-end hardware capacity. +### Local Connection Failure Control + +On 2026-10-08, Windows headless Chrome 154 ran the actual `HostPeer` and pool +with a moving 640x360 VP8 source. mDNS host candidates supported one producer +and two receivers. Discarding only the producer's ICE candidates triggered its +eight-second timeout; both existing P2P connections then delivered 30 ordinary +decoded frames each, retaining the source and no producer groups. + +This loopback-origin diagnostic does not test public-origin permission, +Chrome/Firefox LAN interoperability or public-path quality. In +[#465](https://github.com/TNTcraftHIM/Piik/issues/465), the successful repeat +changed permission and IP masking together. Neither is an isolated cause; +[HTTP App discovery](https://developer.chrome.com/blog/local-network-access) +and [WebRTC connectivity](https://chromestatus.com/feature/5065884686876672) +need separate evidence. This control is not a release acceptance gate. + ### Balanced Startup And Recovery Serial Chrome 152 checks on 2026-09-25 reproduced two composition defects in diff --git a/docs/todo.md b/docs/todo.md index dda565d39..a2132b116 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -127,9 +127,19 @@ of a reporter's cause; research owns the completed experiments and their limits. [#453](https://github.com/TNTcraftHIM/Piik/issues/453) has no matched endpoint reports to distinguish network reachability from a product failure. The v1.6.3 candidate-timeout report lacks share-start and SDP/ICE evidence. - A self-hosted SFU report supplies only a screenshot of a third-party diagnosis; - obtain the original Viewer decode/`route-ready` events and matching Server - revision before attributing timeout to the committed-publication flag. + [#464](https://github.com/TNTcraftHIM/Piik/issues/464) adds Server/Host excerpts + to the self-hosted SFU screenshot report, but still lacks the matched Viewer + ICE, decode and `route-ready` trace. Healthy publication ingress and a bound + subscription do not establish delivered frames; the committed-publication + flag alone cannot identify why the candidate timed out. + [#465](https://github.com/TNTcraftHIM/Piik/issues/465) reports Chrome/Firefox + LAN asymmetry, local-pool timeout and separate public-path quality loss. + Its successful repeat changed both local-network permission and IP masking. + [Local controls](./research/browser-local-encoding-pool.md#local-connection-failure-control) + retain P2P through producer timeout; isolate the settings and compare paired + reports before changing routing or advising users to disable IP masking. + [#463](https://github.com/TNTcraftHIM/Piik/issues/463) lacks the Browser-only + Mac/phone attempt details needed to distinguish capture, entry and media failure. - [ ] **Camera and Host microphone device coverage.** Check real audio levels, echo, device replacement/native mixing and phone camera permission, orientation and background behavior. Include OBS virtual-camera/audio-input sound quality From 171dd1c757602f05940e10109c0ab14a03ea9756 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:38:23 +0800 Subject: [PATCH 041/262] docs(transport): record ICE restart composition boundary --- docs/research/nat-traversal.md | 22 ++++++++++++++++++++++ docs/todo.md | 5 ++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index ab1336822..df57a32f7 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -450,6 +450,28 @@ unilateral direct address are separate facts. Underlay setup must fit the existi operation deadline. Local cold setup of about 3.3 seconds for Tailcat versus 0.2 seconds for Pion disabled STUN/gateway mapping; it is not a WAN startup comparison. +A subsequent actual ICE-restart control on 2026-10-08 exposed a composition +failure. After the same nine-second kernel blackout, ordinary Pion resumed +media, but default Tailcat sometimes sent eight inner Binding requests without +receiving any, then remained unavailable through Pion's failed state. In the +captured failure, outer discovery resumed about 2.4 seconds after restoration, +after the inner check budget had already been spent. Pion's +[candidate-pair check budget](https://github.com/pion/ice/blob/v4.4.0/agent.go#L790) +and Magicsock's +[rate-limited discovery](https://github.com/tailscale/tailscale/blob/a0e471a35b8f/wgengine/magicsock/endpoint.go#L1421) +are independent; a restored outer path does not restart exhausted inner checks. +Keeping the accepted flow's listener open did not resolve the failure. A new +ICE generation after underlay recovery delivered three fresh packets in about +63 ms. Increasing only the fixture's Binding budget to 64 recovered in about +1.9 seconds; both controls retained the same sockets, keys and NAT mappings. +These distinguish underlay/check scheduling from permanent resource retirement. +The isolated authorization and feedback patches retain the same restart failure. +The pinned default restart assertion remains a reproducible integration failure, +not a passing gate or a diagnosis of an existing Piik user report. Do not widen +product timeouts or add diagnostic Ping polling to hide it. An actual local +ICE restart on a shared tunnel also preserved continuous sibling audio/video; +that healthy-network control does not establish impaired-network recovery. + Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof required fresh bidirectional delivery after DERP closed. One side reporting a diff --git a/docs/todo.md b/docs/todo.md index a2132b116..7b2b24026 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -66,7 +66,10 @@ Implementation needs a concrete useful result within the product model. Kernel checks now include Piik's actual Native engine, shared predictor, real gateway mapping and verified multi-destination netcheck alongside dual-stack, mapping replacement and bounded loss/reordering. They still do not - establish extra direct reachability. Keep the + establish extra direct reachability. Actual ICE restart after a kernel blackout + also exposes an inner-check/underlay-recovery timing conflict; the default + Tailcat gate does not reliably pass. A wider fixture budget identifies the cause, + not an accepted timeout change. Keep the experiment separate from product routing; the P2P evidence work below owns the network/interop acceptance boundary. - [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) From 65ccb5e4f7a2116009efb78db1ae10eff5c69e28 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:16:00 +0800 Subject: [PATCH 042/262] docs(transport): qualify underlay rediscovery candidate --- docs/research/nat-traversal.md | 29 ++++++++++++++++++++++++++++- docs/todo.md | 8 ++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index df57a32f7..3218bc02b 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -323,7 +323,10 @@ Pion ICE. Retired mapping-cache reproductions remain controls, not the baseline. [Tailcat](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/README.md) provides an upstream Go reuse candidate: magicsock plus WireGuard, with DERP bootstrap -and relay fallback. Its Browser path at that pin is relay-only. In the isolated +and relay fallback. Its Browser path at that pin is relay-only. Tailscale's open +[Browser P2P proposal](https://github.com/tailscale/tailscale/issues/21027) uses +WebRTC because the Browser cannot expose raw UDP; it is not an implemented +Browser Magicsock hole-puncher. In the isolated Linux kernel NAT comparison on 2026-10-05, both stacks connected directly through ordinary/ordinary and open/hard gateways in either direction. Neither delivered fresh direct payloads through ordinary/hard, hard/ordinary, hard/hard or blocked @@ -472,6 +475,30 @@ product timeouts or add diagnostic Ping polling to hide it. An actual local ICE restart on a shared tunnel also preserved continuous sibling audio/video; that healthy-network control does not establish impaired-network recovery. +A narrower control kept a three-second recovery window and used Pion's existing +`SetICEMaxBindingRequests(16)`. The nine-second blackout recovered in about +1.9 seconds across three default-library repetitions, but changing the outage +to eleven seconds produced no media before the same deadline. The isolated +authorization/feedback candidate reproduced both outcomes. Other checked outage +phases took up to about 2.9 seconds. Increasing the inner request count therefore +does not ensure recovery inside Piik's current opportunity. Explicitly retiring +that failed flow after three seconds preserved a sibling's continuous audio/video +under the race detector. These fixture windows start after network restoration; +they do not run Piik's complete disconnect-grace, restart and rebuild schedule. +A separate one-shot `Client.DiscoPing` before ICE restart also retained failures +with the default inner budget, despite a successful diagnostic reply. +That diagnostic response alone does not prove bidirectional encrypted media +readiness. Neither counterfactual is an accepted transport adapter. + +An isolated follow-up mirrored that discovery call on Tailcat's Server for the +existing fixture peer. Calling both sides once before ICE restart recovered the +nine- and eleven-second controls twice under the race detector, within the same +three-second budget including rediscovery and SDP exchange. Pion retained its +default request count. This supports coordinating underlay revalidation with the +existing restart owner instead of widening deadlines. The Server hook is not an +upstream API, and this probe does not validate concurrent revocation, failed +rediscovery, the complete Piik recovery schedule or additional NAT reachability. + Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof required fresh bidirectional delivery after DERP closed. One side reporting a diff --git a/docs/todo.md b/docs/todo.md index 7b2b24026..55b0ecb3f 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -68,8 +68,12 @@ Implementation needs a concrete useful result within the product model. dual-stack, mapping replacement and bounded loss/reordering. They still do not establish extra direct reachability. Actual ICE restart after a kernel blackout also exposes an inner-check/underlay-recovery timing conflict; the default - Tailcat gate does not reliably pass. A wider fixture budget identifies the cause, - not an accepted timeout change. Keep the + Tailcat gate does not reliably pass. A wider fixture budget identifies the cause; + a larger request count within the unchanged three-second opportunity still + fails at another outage phase. Explicit bilateral rediscovery recovers the + tested phases with default Pion, but requires an experimental Server hook; + its cancellation/revocation and complete recovery lifecycle need validation. + No timeout change or transport integration is accepted. Keep the experiment separate from product routing; the P2P evidence work below owns the network/interop acceptance boundary. - [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) From 092b83a626d56e77e61e5c83f0ca1385d0c266cf Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:42:38 +0800 Subject: [PATCH 043/262] docs(transport): record rediscovery lifecycle evidence --- docs/research/nat-traversal.md | 18 ++++++++++++------ docs/todo.md | 3 ++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 3218bc02b..009783147 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -492,12 +492,18 @@ readiness. Neither counterfactual is an accepted transport adapter. An isolated follow-up mirrored that discovery call on Tailcat's Server for the existing fixture peer. Calling both sides once before ICE restart recovered the -nine- and eleven-second controls twice under the race detector, within the same -three-second budget including rediscovery and SDP exchange. Pion retained its -default request count. This supports coordinating underlay revalidation with the -existing restart owner instead of widening deadlines. The Server hook is not an -upstream API, and this probe does not validate concurrent revocation, failed -rediscovery, the complete Piik recovery schedule or additional NAT reachability. +nine- and eleven-second controls in three retained race-detector repetitions. +The unchanged three-second budget includes rediscovery and SDP exchange. Pion +retained its default request count. This supports coordinating underlay revalidation with the +existing restart owner instead of widening deadlines. Separate repeated checks +cover both callers' pending cancellation/deadline, revocation during a Server +probe, and unknown-peer rejection while sibling audio/video continues. Caller +cancellation ends the wait, not the upstream ping packets: existing five-second +discovery and ten-second TSMP timers bound their callback retention. +The opt-in fixture retains its isolated Server patch and leaves the default +failure controls intact. This hook is not an upstream API; these checks do not +establish room-authorized integration, the complete Piik recovery schedule or +additional NAT reachability. Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof diff --git a/docs/todo.md b/docs/todo.md index 55b0ecb3f..ab5b19a81 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -72,7 +72,8 @@ Implementation needs a concrete useful result within the product model. a larger request count within the unchanged three-second opportunity still fails at another outage phase. Explicit bilateral rediscovery recovers the tested phases with default Pion, but requires an experimental Server hook; - its cancellation/revocation and complete recovery lifecycle need validation. + bounded cancellation, revocation and sibling isolation pass repeated checks. + Room-authorized integration and the complete recovery lifecycle remain unverified. No timeout change or transport integration is accepted. Keep the experiment separate from product routing; the P2P evidence work below owns the network/interop acceptance boundary. From 8dba7b23001a43d5dda9e20d4dbab04ad546d312 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:18:04 +0800 Subject: [PATCH 044/262] fix(app): retain public invitation failure details --- docs/guide/troubleshooting.md | 5 +-- docs/guide/troubleshooting.zh-CN.md | 5 +-- .../research/cross-platform-client-runtime.md | 15 ++++++-- docs/todo.md | 3 ++ internal/app/publictunnel/process.go | 32 ++++++++++++++--- internal/app/publictunnel/process_test.go | 35 ++++++++++++++++++- 6 files changed, 82 insertions(+), 13 deletions(-) diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index e13c3544c..7b0136270 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -210,8 +210,9 @@ from a page that opens but reports **No media route available**. 1. **Viewers: confirm the link.** Ask the host to check that the App is running and the computer is awake, then send the current invitation. Use a new link after the App restarts; the old address may no longer work. -2. **Hosts: read the terminal error.** Check that the complete App package was - extracted. Save the full startup error and try again with **Debug launch** +2. **Hosts: read the failure details.** Check that the complete App package was + extracted. Save the full error from the launch page or terminal, including + any detail after the exit code, and try again with **Debug launch** enabled. The App retries early tunnel-process exits within its startup deadline. 3. **Compare networks.** Try a phone hotspot. Check whether DNS, a proxy or a firewall blocks Cloudflare Tunnel; adjust only relevant rules. Ask the diff --git a/docs/guide/troubleshooting.zh-CN.md b/docs/guide/troubleshooting.zh-CN.md index d733b2e16..bfaccb219 100644 --- a/docs/guide/troubleshooting.zh-CN.md +++ b/docs/guide/troubleshooting.zh-CN.md @@ -179,8 +179,9 @@ App 的 **公网邀请** 通过 Cloudflare Tunnel 提供临时网页地址。 1. **观众先确认链接。** 请房主确认 App 仍在运行、电脑未休眠,并发送当前邀请。 App 重新启动后应使用新链接,旧地址可能失效。 -2. **房主查看终端原因。** 确认已完整解压 App 程序包;启动失败时,保留终端的完整错误, - 开启 **诊断启动** 后再试。App 会在启动时限内重试提前退出的隧道进程。 +2. **房主查看错误详情。** 确认已完整解压 App 程序包;保留启动页面或终端的完整错误, + 包括退出码后的详细信息。开启 **诊断启动** 后再试。 + App 会在启动时限内重试提前退出的隧道进程。 3. **对照网络。** 可用手机热点重试,检查 DNS、代理或防火墙是否阻止 Cloudflare Tunnel。 只调整相关规则;学校或公司的网络应联系管理员。 diff --git a/docs/research/cross-platform-client-runtime.md b/docs/research/cross-platform-client-runtime.md index 1af2c4ae1..125dc0a74 100644 --- a/docs/research/cross-platform-client-runtime.md +++ b/docs/research/cross-platform-client-runtime.md @@ -167,9 +167,11 @@ the existing 30-second startup budget. This affects HTTP/WebSocket control only; media routes, tunnel identity and ordered shutdown keep their existing owners. DNS failures, API rejection and networks blocking both transports can still fail. The preceding Quick Tunnel allocation is a single HTTPS request with a 15-second -dependency timeout; failure exits before transport fallback applies. The App's -pre-readiness exit message and exit code do not identify that cause. Match the -same run's `public-tunnel` dependency diagnostics before classifying a report. +dependency timeout; failure exits before transport fallback applies. The exit +code alone does not identify that cause. An abnormal exit now retains the latest +sanitized dependency error or plain stderr detail in the existing failure result. +This is the last reported observation, not a classification or a complete log; +match the same run's `public-tunnel` diagnostics when the detail is insufficient. Piik retries a child that exits before readiness, up to four starts with a one-second pause between starts, all within the same 30-second acquisition deadline. The previous child, output readers and temporary config retire before @@ -190,6 +192,13 @@ own the dependency behavior. An early-exit fixture confirms fourth-start recover and bounded exhaustion; these checks do not identify a reporter's API or network failure or guarantee four starts when earlier attempts consume the deadline. +A 2026-10-08 Windows check against a deliberately refused local allocation API +confirmed that cloudflared 2026.8.3 emits its final allocation error as plain +stderr even with `--output json`. A JSON-only error reader therefore misses it. +The process boundary retains that detail without changing retries or shutdown; +normal cancellation still stays clean, and late cleanup cannot erase an already +observed failure. The launcher applies its existing redaction and display limit. + The public App log in [#452](https://github.com/TNTcraftHIM/Piik/issues/452), reviewed on 2026-10-07, records a QUIC timeout followed by a TCP port-7844 connect timeout within that deadline. Thus fallback ran; the report does not diff --git a/docs/todo.md b/docs/todo.md index ab5b19a81..d15c4cdad 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -164,6 +164,9 @@ of a reporter's cause; research owns the completed experiments and their limits. [#452](https://github.com/TNTcraftHIM/Piik/issues/452) records both QUIC and TCP connection timeouts; compare the same endpoint with/without its accelerator. That startup log contains no evidence for its separate H.264 report. + [#466](https://github.com/TNTcraftHIM/Piik/issues/466) supplies only an early-exit + screenshot. Retaining dependency failure details improves the next report; + it does not establish or resolve this attempt's cause. - [ ] **Viewer interruptions.** Obtain paired endpoint/upstream-relay diagnostics for established viewing returning to P2P connecting, including [#429](https://github.com/TNTcraftHIM/Piik/issues/429)'s SFU-to-P2P dropout. diff --git a/internal/app/publictunnel/process.go b/internal/app/publictunnel/process.go index d1f4f6c09..749c66ec0 100644 --- a/internal/app/publictunnel/process.go +++ b/internal/app/publictunnel/process.go @@ -43,8 +43,9 @@ type Process struct { done chan struct{} once sync.Once - mu sync.Mutex - err error + mu sync.Mutex + err error + lastDetail string } func PackagedExecutable() string { @@ -166,7 +167,15 @@ func startAttempt(parent context.Context, executable, localOrigin string) (*Proc var connectOnce sync.Once var outputDone sync.WaitGroup outputDone.Add(2) - readOutput := func(reader io.Reader) { + recordDetail := func(text string) { + // One line is bounded by the scanner. Redact before UI truncation. + if text = diagnostics.SafeText(strings.TrimSpace(text)); text != "" { + process.mu.Lock() + process.lastDetail = text + process.mu.Unlock() + } + } + readOutput := func(reader io.Reader, stderr bool) { defer outputDone.Done() scanner := bufio.NewScanner(reader) scanner.Buffer(make([]byte, 4096), maxLogLineBytes) @@ -178,8 +187,16 @@ func startAttempt(parent context.Context, executable, localOrigin string) (*Proc Error string `json:"error"` } if json.Unmarshal([]byte(line), &entry) != nil { + // cloudflared's CLI also returns plain stderr errors, including + // Quick Tunnel allocation failures before a transport exists. + if stderr { + recordDetail(line) + } slog.Debug("public-tunnel", "event", "dependency-output", "message", diagnostics.SafeText(line)) } else if entry.Level == "error" || entry.Level == "warn" || entry.Level == "fatal" { + if entry.Level != "warn" { + recordDetail(strings.Join([]string{entry.Message, entry.Error}, "\n")) + } slog.Debug("public-tunnel", "event", "dependency-output", "level", entry.Level, "message", diagnostics.SafeText(entry.Message), "cause", diagnostics.SafeText(entry.Error)) } @@ -194,8 +211,8 @@ func startAttempt(parent context.Context, executable, localOrigin string) (*Proc slog.Debug("public-tunnel", "event", "output-read-failed", diagnostics.Error(err)) } } - go readOutput(stdout) - go readOutput(stderr) + go readOutput(stdout, false) + go readOutput(stderr, true) go func() { waitErr := child.Wait() // Classify the observed exit before draining logs: later owner cleanup @@ -209,6 +226,11 @@ func startAttempt(parent context.Context, executable, localOrigin string) (*Proc // Wait closes the pipe readers; join them before App closes its recorder. // Do not wait before Wait: that would bypass its bounded pipe shutdown. outputDone.Wait() + process.mu.Lock() + if process.err != nil && process.lastDetail != "" { + process.err = fmt.Errorf("%w\nlast public invitation detail: %s", process.err, process.lastDetail) + } + process.mu.Unlock() _ = os.Remove(configPath) close(process.done) }() diff --git a/internal/app/publictunnel/process_test.go b/internal/app/publictunnel/process_test.go index f5d7bb3c7..23322c9a3 100644 --- a/internal/app/publictunnel/process_test.go +++ b/internal/app/publictunnel/process_test.go @@ -19,6 +19,11 @@ import ( func TestMain(tests *testing.M) { if mode := os.Getenv("PIIK_TUNNEL_FIXTURE"); mode != "" { + if mode == "plain-exit" { + fmt.Fprintln(os.Stderr, "failed to request quick Tunnel: connect timeout token=fixture-secret") + fmt.Fprintln(os.Stdout, "trailing ordinary output") + os.Exit(7) + } if record := os.Getenv("PIIK_TUNNEL_FIXTURE_ATTEMPTS"); record != "" { previous, _ := os.ReadFile(record) var paths []string @@ -46,7 +51,10 @@ func TestMain(tests *testing.M) { } failures, _ := strconv.Atoi(os.Getenv("PIIK_TUNNEL_FIXTURE_FAILURES")) if len(paths) < failures { - fmt.Fprintln(os.Stderr, "fixture allocation unavailable") + fmt.Fprintln(os.Stderr, `{"level":"error","message":"older recovered failure"}`) + fmt.Fprintln(os.Stderr, `{"level":"error","message":"fixture allocation unavailable","error":"connect timeout token=fixture-secret"}`) + fmt.Fprintln(os.Stderr, `{"level":"warn","message":"cleanup warning"}`) + fmt.Fprintln(os.Stderr, `{"level":"info","message":"cleanup complete"}`) code, _ := strconv.Atoi(os.Getenv("PIIK_TUNNEL_FIXTURE_EXIT_CODE")) os.Exit(code) } @@ -83,6 +91,23 @@ func TestMain(tests *testing.M) { os.Exit(tests.Run()) } +func TestStartupRetainsPlainDependencyFailure(t *testing.T) { + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + t.Setenv("PIIK_TUNNEL_FIXTURE", "plain-exit") + process, err := startAttempt(t.Context(), executable, "http://127.0.0.1:8787") + var failure *exec.ExitError + if process != nil || !errors.Is(err, errStartupExited) || !errors.As(err, &failure) || failure.ExitCode() != 7 { + t.Fatalf("plain dependency failure lost its exit status: %v, %v", process, err) + } + if !strings.Contains(err.Error(), "failed to request quick Tunnel: connect timeout token=[redacted]") || + strings.Contains(err.Error(), "fixture-secret") || strings.Contains(err.Error(), "ordinary output") { + t.Fatalf("plain dependency failure was omitted, unsafe or overwritten: %v", err) + } +} + func TestStartupRetriesOnlyRetiredPreReadyChildren(t *testing.T) { executable, err := os.Executable() if err != nil { @@ -120,6 +145,11 @@ func TestStartupRetriesOnlyRetiredPreReadyChildren(t *testing.T) { if !errors.As(err, &failure) || failure.ExitCode() != sample.exitCode { t.Fatalf("latest dependency exit was lost: %v", err) } + if !strings.Contains(err.Error(), "fixture allocation unavailable\nconnect timeout token=[redacted]") || + strings.Contains(err.Error(), "fixture-secret") || strings.Contains(err.Error(), "older recovered failure") || + strings.Contains(err.Error(), "cleanup") { + t.Fatalf("latest sanitized dependency cause was lost or replaced: %v", err) + } } } assertRetiredAttempts(t, record, 4) @@ -318,6 +348,9 @@ func TestCloseKeepsAnObservedFailureWhileLogsDrain(t *testing.T) { if !errors.As(err, &failure) || failure.ExitCode() != 7 { t.Fatalf("late owner cancellation lost the observed dependency failure: %v", err) } + if !strings.Contains(err.Error(), "fixture exit failure") { + t.Fatalf("late output lost the dependency detail: %v", err) + } case <-time.After(2 * time.Second): t.Fatal("tunnel did not retire after its logs drained") } From 7a370922f521882e83c803a79c703e2e3c4c443f Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:18:09 +0800 Subject: [PATCH 045/262] docs(transport): qualify discovery-retained recovery evidence --- docs/research/nat-traversal.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 009783147..ae497bf8d 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -505,6 +505,22 @@ failure controls intact. This hook is not an upstream API; these checks do not establish room-authorized integration, the complete Piik recovery schedule or additional NAT reachability. +A discovery-retained control on 2026-10-08 addresses a limit of closing DERP: +closure also removes its endpoint exchange. The upstream relay's packet-class +counters distinguish discovery from WireGuard traffic. A 500 ms window with +connected WebRTC, at least three fresh tagged SRTP packets and no non-discovery +relay packets proves direct media while keeping endpoint exchange available. +Ordinary NAT passes that proof; blocked UDP correctly remains relay-only. Both +orientations of the hard-NAT pair deliver through DERP without a proved direct +window in 20 seconds. After either gateway mapping changes, media returns in +about six seconds. Default-library repetitions do not prove direct recovery in +20 seconds. The combined isolated candidate proves it once per direction at +about 17 seconds, but neither of two further repetitions per direction does. +This variation does not establish a repeatable candidate advantage or an extra +reachable class. The control makes no explicit rediscovery call; its passing +result validates the measurement and available-media baseline, not universal +direct recovery. It does not run Piik's complete reconnect schedule. + Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof required fresh bidirectional delivery after DERP closed. One side reporting a From 90c9be35184c89503b19b08d9460bf91994dc87a Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:47:36 +0800 Subject: [PATCH 046/262] docs(transport): clarify recovery and acceptance limits --- docs/research/nat-traversal.md | 89 ++++++++++++++++------------------ docs/todo.md | 9 +++- 2 files changed, 48 insertions(+), 50 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index ae497bf8d..e5bca61d2 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -453,57 +453,50 @@ unilateral direct address are separate facts. Underlay setup must fit the existi operation deadline. Local cold setup of about 3.3 seconds for Tailcat versus 0.2 seconds for Pion disabled STUN/gateway mapping; it is not a WAN startup comparison. -A subsequent actual ICE-restart control on 2026-10-08 exposed a composition -failure. After the same nine-second kernel blackout, ordinary Pion resumed -media, but default Tailcat sometimes sent eight inner Binding requests without -receiving any, then remained unavailable through Pion's failed state. In the -captured failure, outer discovery resumed about 2.4 seconds after restoration, -after the inner check budget had already been spent. Pion's +Actual ICE-restart controls on 2026-10-08 exposed a composition failure. +After a nine-second kernel blackout, ordinary Pion resumed media, but default +Tailcat sometimes exhausted eight inner Binding requests before outer discovery +recovered about 2.4 seconds after restoration. Pion's [candidate-pair check budget](https://github.com/pion/ice/blob/v4.4.0/agent.go#L790) and Magicsock's [rate-limited discovery](https://github.com/tailscale/tailscale/blob/a0e471a35b8f/wgengine/magicsock/endpoint.go#L1421) -are independent; a restored outer path does not restart exhausted inner checks. -Keeping the accepted flow's listener open did not resolve the failure. A new -ICE generation after underlay recovery delivered three fresh packets in about -63 ms. Increasing only the fixture's Binding budget to 64 recovered in about -1.9 seconds; both controls retained the same sockets, keys and NAT mappings. -These distinguish underlay/check scheduling from permanent resource retirement. -The isolated authorization and feedback patches retain the same restart failure. -The pinned default restart assertion remains a reproducible integration failure, -not a passing gate or a diagnosis of an existing Piik user report. Do not widen -product timeouts or add diagnostic Ping polling to hide it. An actual local -ICE restart on a shared tunnel also preserved continuous sibling audio/video; -that healthy-network control does not establish impaired-network recovery. - -A narrower control kept a three-second recovery window and used Pion's existing -`SetICEMaxBindingRequests(16)`. The nine-second blackout recovered in about -1.9 seconds across three default-library repetitions, but changing the outage -to eleven seconds produced no media before the same deadline. The isolated -authorization/feedback candidate reproduced both outcomes. Other checked outage -phases took up to about 2.9 seconds. Increasing the inner request count therefore -does not ensure recovery inside Piik's current opportunity. Explicitly retiring -that failed flow after three seconds preserved a sibling's continuous audio/video -under the race detector. These fixture windows start after network restoration; -they do not run Piik's complete disconnect-grace, restart and rebuild schedule. -A separate one-shot `Client.DiscoPing` before ICE restart also retained failures -with the default inner budget, despite a successful diagnostic reply. -That diagnostic response alone does not prove bidirectional encrypted media -readiness. Neither counterfactual is an accepted transport adapter. - -An isolated follow-up mirrored that discovery call on Tailcat's Server for the -existing fixture peer. Calling both sides once before ICE restart recovered the -nine- and eleven-second controls in three retained race-detector repetitions. -The unchanged three-second budget includes rediscovery and SDP exchange. Pion -retained its default request count. This supports coordinating underlay revalidation with the -existing restart owner instead of widening deadlines. Separate repeated checks -cover both callers' pending cancellation/deadline, revocation during a Server -probe, and unknown-peer rejection while sibling audio/video continues. Caller -cancellation ends the wait, not the upstream ping packets: existing five-second -discovery and ten-second TSMP timers bound their callback retention. -The opt-in fixture retains its isolated Server patch and leaves the default -failure controls intact. This hook is not an upstream API; these checks do not -establish room-authorized integration, the complete Piik recovery schedule or -additional NAT reachability. +are independent: outer recovery does not restart exhausted inner checks. +Keeping the accepted listener open did not help; a new ICE generation after +underlay recovery delivered three fresh packets in about 63 ms. This distinguishes +check scheduling from permanently retired resources. The isolated authorization +and feedback patches retain this failure. + +The following counterfactuals preserve sockets, keys and NAT mappings. Unless +noted, recovery starts after network restoration; none runs Piik's complete +disconnect-grace, restart and rebuild schedule. + +| Tailcat recovery control | Observation | +| --- | --- | +| Increase only the fixture's Binding request budget to 64 | Media recovered in about 1.9 seconds inside the fixture's eight-second window. This does not accept a product timeout change. | +| 16 Binding requests, three-second recovery window | Nine-second blackout recovered in about 1.9 seconds across three default-library repetitions; eleven seconds did not recover in budget. Other checked phases reached about 2.9 seconds. The authorization/feedback candidate reproduced both outcomes. | +| One client-side `DiscoPing` before default-Pion restart | Failures remained even after a diagnostic reply; that reply does not prove bidirectional encrypted media readiness. | +| One probe on each side before default-Pion restart | Nine- and eleven-second blackouts recovered in three race-detector repetitions, including both probes and SDP exchange inside three seconds. This requires an isolated, unsupported Server hook. | +| Same bilateral probe begins while the network is still down | After eight seconds of blackout, restoration one or two seconds into the probe still left it waiting until its three-second deadline in repeated race checks. Post-restoration controls continued to pass. | + +The last control limits the bilateral proposal: a +[`DiscoPing` call](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/tailcat.go#L2258) +waits for its own diagnostic reply, not an ongoing connection-readiness event. +Later network restoration cannot guarantee that a lost probe completes. +The earlier passing phase therefore does not establish a universal recovery +barrier. These controls use fixture SDP exchange; they do not diagnose the +product's Trickle ICE or an existing user's failure. + +Separate checks preserve continuous sibling audio/video through a local ICE +restart, failed-flow retirement, both callers' pending cancellation/deadline, +revocation during a Server probe and unknown-peer rejection. Canceling a probe +ends the caller's wait, not upstream packets: existing five-second discovery and +ten-second TSMP timers bound callback retention. These local checks do not prove +impaired-network recovery. + +The opt-in experiment retains the failing default and pre-restoration assertions. +It is not an all-green acceptance gate or a supported transport adapter. Do not +widen product timeouts or add diagnostic Ping polling to hide these failures. +Room-authorized integration and the complete recovery lifecycle remain unverified. A discovery-retained control on 2026-10-08 addresses a limit of closing DERP: closure also removes its endpoint exchange. The upstream relay's packet-class diff --git a/docs/todo.md b/docs/todo.md index d15c4cdad..0266bfcfa 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -15,6 +15,9 @@ Git/PRs own completed history. coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. + Windows full Go acceptance needs a runner that permits its test executables; + the current run stopped before the Browser-launcher package tests with an OS + access denial. Separate native checks and Linux checks do not close that gap. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance are complete. Integrate through the next owner-approved publishing merge and verify the separate workflow with its original CI artifacts. Mirror failure @@ -71,8 +74,10 @@ Implementation needs a concrete useful result within the product model. Tailcat gate does not reliably pass. A wider fixture budget identifies the cause; a larger request count within the unchanged three-second opportunity still fails at another outage phase. Explicit bilateral rediscovery recovers the - tested phases with default Pion, but requires an experimental Server hook; - bounded cancellation, revocation and sibling isolation pass repeated checks. + tested post-restoration phases with default Pion, but requires an experimental + Server hook and fails when called before network restoration. Bounded + cancellation, revocation and sibling isolation pass repeated checks; + a diagnostic reply is not a general underlay-readiness contract. Room-authorized integration and the complete recovery lifecycle remain unverified. No timeout change or transport integration is accepted. Keep the experiment separate from product routing; the P2P evidence work below owns From 92825f084c5b79a79c830ae836461b358da06964 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:20:58 +0800 Subject: [PATCH 047/262] docs(transport): distinguish failed probes from failed media --- docs/research/nat-traversal.md | 6 ++++-- docs/todo.md | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index e5bca61d2..55b239611 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -477,13 +477,15 @@ disconnect-grace, restart and rebuild schedule. | One client-side `DiscoPing` before default-Pion restart | Failures remained even after a diagnostic reply; that reply does not prove bidirectional encrypted media readiness. | | One probe on each side before default-Pion restart | Nine- and eleven-second blackouts recovered in three race-detector repetitions, including both probes and SDP exchange inside three seconds. This requires an isolated, unsupported Server hook. | | Same bilateral probe begins while the network is still down | After eight seconds of blackout, restoration one or two seconds into the probe still left it waiting until its three-second deadline in repeated race checks. Post-restoration controls continued to pass. | +| Lost bilateral probes, retain the existing media connection | In six race checks, both probes expired while WebRTC was already connected; a subsequent observation received fresh SRTP without new SDP, sockets or credentials. The separate three-second observation is not a product recovery extension. | -The last control limits the bilateral proposal: a +These controls limit the bilateral proposal: a [`DiscoPing` call](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/tailcat.go#L2258) waits for its own diagnostic reply, not an ongoing connection-readiness event. Later network restoration cannot guarantee that a lost probe completes. The earlier passing phase therefore does not establish a universal recovery -barrier. These controls use fixture SDP exchange; they do not diagnose the +barrier, and probe expiry does not establish media failure or justify replacing +healthy media. These controls use fixture SDP exchange; they do not diagnose the product's Trickle ICE or an existing user's failure. Separate checks preserve continuous sibling audio/video through a local ICE diff --git a/docs/todo.md b/docs/todo.md index 0266bfcfa..fd812660d 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -75,7 +75,8 @@ Implementation needs a concrete useful result within the product model. a larger request count within the unchanged three-second opportunity still fails at another outage phase. Explicit bilateral rediscovery recovers the tested post-restoration phases with default Pion, but requires an experimental - Server hook and fails when called before network restoration. Bounded + Server hook. Its proposed readiness gate fails when called before network + restoration; probe expiry can coexist with recovered media. Bounded cancellation, revocation and sibling isolation pass repeated checks; a diagnostic reply is not a general underlay-readiness contract. Room-authorized integration and the complete recovery lifecycle remain unverified. From ccf30bc9a34562cb5423139f213edcd29140fd52 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:47:19 +0800 Subject: [PATCH 048/262] fix(diagnostics): distinguish SFU endpoint connectivity --- docs/guide/troubleshooting.md | 3 +++ docs/guide/troubleshooting.zh-CN.md | 2 ++ docs/operations/self-hosting.md | 3 ++- docs/operations/self-hosting.zh-CN.md | 1 + docs/standards/configuration.md | 2 +- docs/todo.md | 13 ++++++++----- internal/server/signal/router_sfu_signal.go | 12 +++++++----- internal/server/signal/router_sfu_signal_test.go | 10 ++++++++++ 8 files changed, 34 insertions(+), 12 deletions(-) diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 7b0136270..350551cfe 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -301,6 +301,9 @@ explains discovery and connection checks in more technical detail. If these steps do not help, collect a [Debug report](../../cmd/piik-app/README.md#diagnostics) from the affected Viewer and, if possible, the Host for the same attempt. Include the time, Piik version and network type when reporting the problem. +For a self-hosted site using SFU, also include the Server report. Successful Host +upload or subscription setup alone does not establish that the Viewer received +and decoded the picture; the reports from both sides help locate the failing step. ### Browser WebRTC restrictions diff --git a/docs/guide/troubleshooting.zh-CN.md b/docs/guide/troubleshooting.zh-CN.md index bfaccb219..7b499f6ea 100644 --- a/docs/guide/troubleshooting.zh-CN.md +++ b/docs/guide/troubleshooting.zh-CN.md @@ -247,6 +247,8 @@ NAT(网络地址转换)让多台设备共用一个上网地址。不同路 仍然失败时,收集出现问题的观众端与房主端同一次尝试的 [诊断报告](../../cmd/piik-app/README.zh-CN.md#诊断),反馈时附上发生时间、Piik 版本和所用网络类型。 +自部署站点使用 SFU 时,也请附上服务端报告。房主上传成功或订阅建立,并不代表观众已经收到并解码画面; +两端报告能帮助判断具体卡在哪一步。 ### 浏览器限制了 WebRTC diff --git a/docs/operations/self-hosting.md b/docs/operations/self-hosting.md index 64389b11d..ce2ea916a 100644 --- a/docs/operations/self-hosting.md +++ b/docs/operations/self-hosting.md @@ -144,7 +144,8 @@ configuration uses P2P media, so participants need a usable UDP path. Add `SFU_UDP_PORT=7882` to `.env`, allow UDP 7882, and restart Piik to enable automatic SFU fallback. When the server is behind NAT, also set `SFU_PUBLIC_IP` -to its reachable public IPv4 address. The same binary provides the fallback. +to its reachable public IPv4 address and forward the same external and internal +UDP port as `SFU_UDP_PORT`. The same binary provides the fallback. Allow server forwarding capacity and outbound bandwidth for Viewers actually using the SFU. The Host must turn off **Privacy mode** before sharing to allow this route. diff --git a/docs/operations/self-hosting.zh-CN.md b/docs/operations/self-hosting.zh-CN.md index f91fa1cfc..1e8a05977 100644 --- a/docs/operations/self-hosting.zh-CN.md +++ b/docs/operations/self-hosting.zh-CN.md @@ -130,6 +130,7 @@ TCP 8787 仅供本机反向代理访问。STUN 域名需要直接解析到服务 在 `.env` 中添加 `SFU_UDP_PORT=7882`,放行 UDP 7882,再重启 Piik,即可启用自动 SFU 兜底。 如果服务器处于 NAT 后方,还需将 `SFU_PUBLIC_IP` 设置为外部可达的公网 IPv4 地址。 +公网和内部映射的 UDP 端口都应与 `SFU_UDP_PORT` 一致。 这项功能由同一个服务端程序提供。 服务器需要为实际经由 SFU 观看的观众承担转发负载和出口带宽。 房主需要在开始分享前关闭 **隐私模式**,才会允许使用这条线路。 diff --git a/docs/standards/configuration.md b/docs/standards/configuration.md index e121012e0..23afca7a2 100644 --- a/docs/standards/configuration.md +++ b/docs/standards/configuration.md @@ -65,7 +65,7 @@ Automatic SFU fallback runs inside the Hosted process when `SFU_UDP_PORT` is set | --- | --- | | `SFU_UDP_PORT` | Optional UDP media port `1..65535`; unset or blank disables SFU and fixes Privacy mode on for every room. Set `7882` for the standard public listener and allow the Host to choose Privacy mode. | | `SFU_LISTEN_HOST` | IPv4 bind address, default `0.0.0.0`; independent of HTTP `LISTEN_HOST`. Read only when SFU is enabled. | -| `SFU_PUBLIC_IP` | Optional explicit IPv4 advertised-address override for a host behind NAT. Read only when SFU is enabled. | +| `SFU_PUBLIC_IP` | Optional explicit IPv4 advertised-address override for a host behind NAT. It changes the advertised address, not the port: forward the same UDP port as `SFU_UDP_PORT`. Read only when SFU is enabled. | | `SFU_ONLY` | `false` by default. `true` requires `SFU_UDP_PORT` and sends every room through SFU, without Peer candidates or background P2P convergence. Applies after Server restart. | SFU control uses the application's authenticated signaling connection. No diff --git a/docs/todo.md b/docs/todo.md index fd812660d..950ef10bb 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -141,11 +141,14 @@ of a reporter's cause; research owns the completed experiments and their limits. [#453](https://github.com/TNTcraftHIM/Piik/issues/453) has no matched endpoint reports to distinguish network reachability from a product failure. The v1.6.3 candidate-timeout report lacks share-start and SDP/ICE evidence. - [#464](https://github.com/TNTcraftHIM/Piik/issues/464) adds Server/Host excerpts - to the self-hosted SFU screenshot report, but still lacks the matched Viewer - ICE, decode and `route-ready` trace. Healthy publication ingress and a bound - subscription do not establish delivered frames; the committed-publication - flag alone cannot identify why the candidate timed out. + [#464](https://github.com/TNTcraftHIM/Piik/issues/464) and the supplied report + identify v1.9.2, a NAT-hosted Server, Native Host, and the same timeout under + `SFU_ONLY`. They still lack the matched Viewer ICE, decode and `route-ready` + trace. The cited Host failure follows the operation deadline; healthy ingress + and a bound subscription do not establish delivered frames or its cause. + The local Native H264/SFU-only control passes, but does not reproduce the + reporter's network. Correlate transport-state diagnostics with the affected + Viewer before changing admission or recovery. [#465](https://github.com/TNTcraftHIM/Piik/issues/465) reports Chrome/Firefox LAN asymmetry, local-pool timeout and separate public-path quality loss. Its successful repeat changed both local-network permission and IP masking. diff --git a/internal/server/signal/router_sfu_signal.go b/internal/server/signal/router_sfu_signal.go index ceff44b73..b817be574 100644 --- a/internal/server/signal/router_sfu_signal.go +++ b/internal/server/signal/router_sfu_signal.go @@ -225,7 +225,7 @@ func (r *router) handleSfuMediaEvent(event sfu.MediaEvent) { peer, connected := r.connectedPeer(event.Fence.RoomID, peerID) snapshot := rm.controller.Snapshot() participant := authenticatedRouteParticipant{roomID: event.Fence.RoomID, role: role, peerID: peerID, sessionID: peer.SessionID} - _, revision := sfuConnectionPhase(snapshot, role, peerID, event.Fence.PublicationGeneration, event.ConnectionID) + phase, revision := sfuConnectionPhase(snapshot, role, peerID, event.Fence.PublicationGeneration, event.ConnectionID) message := protocol.SfuSignalMessage{Type: "sfu-signal", Kind: "candidate", Revision: protocol.Int(revision), PublicationGeneration: event.Fence.PublicationGeneration, ConnectionID: event.ConnectionID} if !connected && role == protocol.RoleHost && event.State == webrtc.PeerConnectionStateFailed { publication := snapshot.HostPublication @@ -245,11 +245,13 @@ func (r *router) handleSfuMediaEvent(event sfu.MediaEvent) { r.hooks.sendToSession(participant.sessionID, message) return } - if event.State == webrtc.PeerConnectionStateFailed { - r.failSfuConnection(participant, message) - return - } if event.State != webrtc.PeerConnectionStateUnknown { + r.debug(participant.roomID, "sfu-transport-state", + "participant", r.debugPeer(participant.roomID, participant.peerID), + "role", role, "phase", phase, "revision", revision, "state", event.State.String()) + if event.State == webrtc.PeerConnectionStateFailed { + r.failSfuConnection(participant, message) + } return } if event.Candidate != nil { diff --git a/internal/server/signal/router_sfu_signal_test.go b/internal/server/signal/router_sfu_signal_test.go index 7b0487b97..f8068c576 100644 --- a/internal/server/signal/router_sfu_signal_test.go +++ b/internal/server/signal/router_sfu_signal_test.go @@ -66,6 +66,7 @@ func TestEmbeddedSfuSignalsRequireExactAdmittedConnectionAndCurrentSession(t *te } func TestEmbeddedSfuAuthenticatedWebSocketCarriesActualRTP(t *testing.T) { + diagnostics := captureRouteDebug(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() check := func(err error) { @@ -216,6 +217,15 @@ func TestEmbeddedSfuAuthenticatedWebSocketCarriesActualRTP(t *testing.T) { if !waitFor(t, 3*time.Second, func() bool { return received.Load() >= 3 }) { t.Fatal("authenticated signaling did not produce forwarded video RTP") } + if !waitFor(t, time.Second, func() bool { + return diagnostics.calledWith(h.room.RoomID, "sfu-transport-state", map[string]any{ + "role": "host", "phase": "prepare", "revision": int64(hostConfig.Revision), "state": "connected", + }) && diagnostics.calledWith(h.room.RoomID, "sfu-transport-state", map[string]any{ + "role": "viewer", "phase": "prepare", "revision": int64(viewerConfig.Revision), "state": "connected", + }) + }) { + t.Fatal("SFU diagnostics did not distinguish connected ingress and egress before route commit") + } h.locked(func() { room, _ := h.server.router.rooms.Get(h.room.RoomID) if room.controller.Operation() == nil { From fad248f73e07498615e6e20d0483b7a565429306 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:21:04 +0800 Subject: [PATCH 049/262] docs: consolidate transport recovery evidence --- docs/research/nat-traversal.md | 25 ++++++++++++++++++++-- docs/todo.md | 39 +++++++++------------------------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 55b239611..0be016afd 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -488,6 +488,27 @@ barrier, and probe expiry does not establish media failure or justify replacing healthy media. These controls use fixture SDP exchange; they do not diagnose the product's Trickle ICE or an existing user's failure. +A follow-up kernel fixture on 2026-10-09 matches the committed Viewer's two +three-second recovery opportunities: restart ICE, then rebuild only if still +not connected at the first deadline. It retains tunnel identity and outer NAT +mappings, creates a new inner flow for replacement, and keeps default Pion checks +without an explicit discovery hook. Setup consumes its opportunity; fresh SRTP +is checked separately after connection meets the deadline. Post-restoration +controls cover eight-, nine- and eleven-second blackouts; Tailcat also starts +recovery one or two seconds before restoration. All five Tailcat cases pass two +further race-detector repetitions, recovering through restart or replacement; +the ordinary Pion controls also pass. Failure of the first ICE generation alone +therefore does not establish failure of both existing recovery opportunities. + +The fixture's replacement flow repeats an initial datagram until the SDP exchange +confirms listener acceptance, bounded by that same opportunity. A lone lost +preface had blocked setup before ICE could run. This fixture handshake is not an +accepted product protocol or proof of media readiness. Its complete-gathering +SDP exchange also stalls ordinary Pion when started before network restoration; +that result is not Piik's Trickle ICE behavior. The passing controls narrow the +integration question but do not run the actual Viewer scheduler, room authority +or Browser decoder, and do not establish additional direct reachability. + Separate checks preserve continuous sibling audio/video through a local ICE restart, failed-flow retirement, both callers' pending cancellation/deadline, revocation during a Server probe and unknown-peer rejection. Canceling a probe @@ -495,8 +516,8 @@ ends the caller's wait, not upstream packets: existing five-second discovery and ten-second TSMP timers bound callback retention. These local checks do not prove impaired-network recovery. -The opt-in experiment retains the failing default and pre-restoration assertions. -It is not an all-green acceptance gate or a supported transport adapter. Do not +The opt-in experiment retains the failing single-restart and blocking-probe +assertions. It is not an all-green acceptance gate or a supported adapter. Do not widen product timeouts or add diagnostic Ping polling to hide these failures. Room-authorized integration and the complete recovery lifecycle remain unverified. diff --git a/docs/todo.md b/docs/todo.md index 950ef10bb..643031529 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -1,6 +1,6 @@ # Current TODO Ledger -Last reviewed: 2026-10-08 +Last reviewed: 2026-10-09 The current 2.0 preparation includes independent investigation across this ledger. Explicit owner holds remain in force; an investigation does not itself @@ -55,34 +55,15 @@ Implementation needs a concrete useful result within the product model. - [ ] **Native Magicsock.** Go reuse is feasible, but the [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) have not established improved direct reachability. Seek a repeatable benefit - before accepting peer-key authorization/revocation, underlay readiness and - relay operation. The [authorization assessment](./research/nat-traversal.md#peer-authorization-boundary) - reproduces stale WireGuard sessions, late admission and a deeper lazy-peer - creation race. Isolated Tailcat/WireGuard corrections pass bounded checks; - the scoped lookup fence also preserves unrelated concurrent admission. - Supported upstream fixes still need review before product adoption. - The isolated feedback check also requires a supported RTCP - size boundary for the tunnel's MTU, including reordered-history reports and - compound batches; shorter feedback intervals and the checked upstream updates - do not supply it. An isolated upstream repair candidate passes actual encrypted - feedback and race checks; it is not a supported or adopted dependency change. - Kernel checks now include Piik's actual Native engine, shared predictor, - real gateway mapping and verified multi-destination netcheck alongside - dual-stack, mapping replacement and bounded loss/reordering. They still do not - establish extra direct reachability. Actual ICE restart after a kernel blackout - also exposes an inner-check/underlay-recovery timing conflict; the default - Tailcat gate does not reliably pass. A wider fixture budget identifies the cause; - a larger request count within the unchanged three-second opportunity still - fails at another outage phase. Explicit bilateral rediscovery recovers the - tested post-restoration phases with default Pion, but requires an experimental - Server hook. Its proposed readiness gate fails when called before network - restoration; probe expiry can coexist with recovered media. Bounded - cancellation, revocation and sibling isolation pass repeated checks; - a diagnostic reply is not a general underlay-readiness contract. - Room-authorized integration and the complete recovery lifecycle remain unverified. - No timeout change or transport integration is accepted. Keep the - experiment separate from product routing; the P2P evidence work below owns - the network/interop acceptance boundary. + before accepting room-scoped peer keys, flow admission and relay operation. + Product adoption still needs supported + [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) + and RTCP/MTU boundaries, plus verification through the actual Viewer recovery + owner and mixed Browser/Native routes. Keep isolated dependency patches and + recovery counterfactuals outside product routing; a diagnostic reply is not + media-readiness authority. Research owns the experiments and their limits. + No timeout change or transport integration is accepted; the P2P evidence work + below owns the network/interop acceptance boundary. - [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) identifies UDP/storage requirements but establishes no cost saving. Keep the existing P2P-only Demo until a provider and operating boundary are accepted. From d1680e0a2f2dcc9022b11aabe0a905c1aa3f6dd6 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:25:20 +0800 Subject: [PATCH 050/262] fix(media): preserve viewer recovery budget across replacement --- docs/standards/routing-transport.md | 5 ++- src/client/native/native-viewer-peer.ts | 47 +++++++++++--------- src/client/webrtc/viewer-peer.ts | 45 +++++++++++-------- tests/native-viewer-peer.test.ts | 34 +++++++++++++-- tests/viewer-peer-generation.test.ts | 57 +++++++++++++++---------- 5 files changed, 124 insertions(+), 64 deletions(-) diff --git a/docs/standards/routing-transport.md b/docs/standards/routing-transport.md index c84ede8a5..a2a97ac6f 100644 --- a/docs/standards/routing-transport.md +++ b/docs/standards/routing-transport.md @@ -119,8 +119,11 @@ deadline or start an independent recovery loop. When NAT traversal enhancement is enabled, another operation may use the remaining connection-attempt budget after rollback. The waiting display uses the actual server-issued attempt ordinal; it never counts time as an attempted connection. -Committed Browser P2P recovery first restarts ICE on the same connection, then +Committed P2P recovery first restarts ICE on the same connection, then rebuilds that connection if necessary, within the existing two-request budget. +The budget belongs to the current parent and survives physical connection or +Browser/Native backend replacement. Only a connected transport or a new parent +resets it; replacing an unconnected backend cannot renew recovery indefinitely. Capable Native receivers also renegotiate that connection while retaining the encoded source, local playback and downstream edges when its codec/audio shape is unchanged. Pion owns ICE restart; each actual gathering owns fresh supplemental diff --git a/src/client/native/native-viewer-peer.ts b/src/client/native/native-viewer-peer.ts index 5233373e8..aaea00220 100644 --- a/src/client/native/native-viewer-peer.ts +++ b/src/client/native/native-viewer-peer.ts @@ -15,6 +15,8 @@ import { ViewerPeer, type ViewerPeerEvents, type ViewerPeerOptions, + type ViewerRecoveryBudget, + createViewerRecoveryBudget, } from "../webrtc/viewer-peer"; import { iceServersWithNatPrediction, @@ -61,6 +63,10 @@ export class NativeCapableViewerPeer implements ViewerMediaPeer { private readonly edgeCapacity: number, ) { this.currentIceConfig = iceConfig; + this.options = { + ...options, + recoveryBudget: options.recoveryBudget ?? createViewerRecoveryBudget(), + }; } acceptSignal(parentPeerId: string, payload: SignalPayload): Promise { @@ -87,6 +93,9 @@ export class NativeCapableViewerPeer implements ViewerMediaPeer { ) { this.backend?.dispose(); this.backend = null; + if (this.identity && this.identity.parentPeerId !== parentPeerId) { + this.options = { ...this.options, recoveryBudget: createViewerRecoveryBudget() }; + } this.identity = nextIdentity; } if (!this.backend) { @@ -198,8 +207,7 @@ class NativeViewerPeer implements ViewerMediaPeer { private disconnectTimer: number | null = null; private initialConnectionTimer: number | null = null; private recoveryTimer: number | null = null; - private automaticRecoveryRequests = 0; - private recoveryExhaustedReported = false; + private readonly recoveryBudget: ViewerRecoveryBudget; private state: RTCPeerConnectionState = "new"; private localCandidateType: RTCIceCandidateType | null = null; private remoteCandidateType: RTCIceCandidateType | null = null; @@ -236,6 +244,7 @@ class NativeViewerPeer implements ViewerMediaPeer { ) { this.currentIceConfig = iceConfig; this.recoveryOwner = options.recoveryOwner ?? "viewer"; + this.recoveryBudget = options.recoveryBudget ?? createViewerRecoveryBudget(); this.natPredictionEnabled = options.natPredictionEnabled ?? false; this.unsubscribe = client.onEvent((event) => this.onEvent(event)); this.unsubscribeClose = client.onClose(() => { @@ -291,8 +300,9 @@ class NativeViewerPeer implements ViewerMediaPeer { ); if (sent) { this.clearInitialConnectionTimer(); - this.automaticRecoveryRequests = Math.max( - this.automaticRecoveryRequests, + this.recoveryBudget.exhaustedReported = false; + this.recoveryBudget.requests = Math.max( + this.recoveryBudget.requests, rebuild ? VIEWER_MAX_AUTOMATIC_RECOVERY_REQUESTS : 1, ); this.scheduleRecoveryDeadline(); @@ -364,7 +374,6 @@ class NativeViewerPeer implements ViewerMediaPeer { } this.parentPeerId = parentPeerId; this.connectionId = connectionId; - this.resetAutomaticRecovery(); } else if (!retainSource) { this.disposeBridge(); } @@ -478,8 +487,8 @@ class NativeViewerPeer implements ViewerMediaPeer { ); if (this.recoveryOwner === "route") { this.reportRecoveryExhausted(); - } else if (!this.requestRecovery(true)) { - this.reportRecoveryExhausted(); + } else { + this.attemptAutomaticRecovery(true); } return; } @@ -650,41 +659,41 @@ class NativeViewerPeer implements ViewerMediaPeer { this.reportRecoveryExhausted(); return; } - if (!this.requestRecovery(true)) { - this.reportRecoveryExhausted(); - } + this.attemptAutomaticRecovery(true); } - private attemptAutomaticRecovery(): void { + private attemptAutomaticRecovery(forceRebuild = false): void { this.clearInitialConnectionTimer(); if ( this.disposed || this.recoveryTimer !== null || - this.recoveryExhaustedReported || + this.recoveryBudget.exhaustedReported || !this.connectionId || !this.parentPeerId ) { return; } if ( - this.automaticRecoveryRequests >= + this.recoveryBudget.requests >= VIEWER_MAX_AUTOMATIC_RECOVERY_REQUESTS ) { this.reportRecoveryExhausted(); return; } - const rebuild = this.automaticRecoveryRequests > 0; + const rebuild = forceRebuild || this.recoveryBudget.requests > 0; if (!this.events.sendRestartRequest(this.parentPeerId, this.connectionId, rebuild)) { this.reportRecoveryExhausted(); return; } - this.automaticRecoveryRequests += 1; + this.recoveryBudget.requests = rebuild + ? VIEWER_MAX_AUTOMATIC_RECOVERY_REQUESTS + : this.recoveryBudget.requests + 1; this.scheduleRecoveryDeadline(); } private reportRecoveryExhausted(): void { if ( - this.recoveryExhaustedReported || + this.recoveryBudget.exhaustedReported || !this.parentPeerId || !this.connectionId ) return; @@ -693,7 +702,7 @@ class NativeViewerPeer implements ViewerMediaPeer { this.parentPeerId, this.connectionId, ) ?? true; - if (reported) this.recoveryExhaustedReported = true; + if (reported) this.recoveryBudget.exhaustedReported = true; else this.scheduleRecoveryDeadline(); } @@ -721,8 +730,8 @@ class NativeViewerPeer implements ViewerMediaPeer { private resetAutomaticRecovery(): void { this.clearRecoveryTimer(); - this.automaticRecoveryRequests = 0; - this.recoveryExhaustedReported = false; + this.recoveryBudget.requests = 0; + this.recoveryBudget.exhaustedReported = false; } private disposeBridge(): void { diff --git a/src/client/webrtc/viewer-peer.ts b/src/client/webrtc/viewer-peer.ts index 0c1cf5c56..cd3f70138 100644 --- a/src/client/webrtc/viewer-peer.ts +++ b/src/client/webrtc/viewer-peer.ts @@ -53,6 +53,17 @@ export interface ViewerPeerEvents { export interface ViewerPeerOptions { natPredictionEnabled?: boolean; recoveryOwner?: "viewer" | "route"; + recoveryBudget?: ViewerRecoveryBudget; +} + +// Physical replacement does not make an unreachable parent healthy again. +export interface ViewerRecoveryBudget { + requests: number; + exhaustedReported: boolean; +} + +export function createViewerRecoveryBudget(): ViewerRecoveryBudget { + return { requests: 0, exhaustedReported: false }; } export interface ViewerMediaPeer { @@ -89,9 +100,7 @@ export class ViewerPeer implements ViewerMediaPeer { private disconnectTimer: number | null = null; private initialConnectionTimer: number | null = null; private recoveryTimer: number | null = null; - private offerRecoveryAttempts = 0; - private automaticRecoveryRequests = 0; - private recoveryExhaustedReported = false; + private readonly recoveryBudget: ViewerRecoveryBudget; private disposed = false; private currentIceConfig: PeerIceConfig; private localIceCandidates: NatPredictionCandidateEmitter | null = null; @@ -108,6 +117,7 @@ export class ViewerPeer implements ViewerMediaPeer { this.currentIceConfig = iceConfig; this.natPredictionEnabled = options.natPredictionEnabled ?? false; this.recoveryOwner = options.recoveryOwner ?? "viewer"; + this.recoveryBudget = options.recoveryBudget ?? createViewerRecoveryBudget(); } async acceptSignal( @@ -185,8 +195,9 @@ export class ViewerPeer implements ViewerMediaPeer { ); if (sent) { this.clearInitialConnectionTimer(); - this.automaticRecoveryRequests = Math.max( - this.automaticRecoveryRequests, + this.recoveryBudget.exhaustedReported = false; + this.recoveryBudget.requests = Math.max( + this.recoveryBudget.requests, rebuild ? VIEWER_MAX_AUTOMATIC_RECOVERY_REQUESTS : 1, ); this.scheduleRecoveryDeadline(); @@ -248,7 +259,6 @@ export class ViewerPeer implements ViewerMediaPeer { this.parentPeerId !== null && this.parentPeerId !== parentPeerId; this.disposeConnection(); if (parentChanged) { - this.offerRecoveryAttempts = 0; this.resetAutomaticRecovery(); } this.parentPeerId = parentPeerId; @@ -396,7 +406,6 @@ export class ViewerPeer implements ViewerMediaPeer { throw new Error("signal send rejected"); } this.clearRecoveryTimer(); - this.offerRecoveryAttempts = 0; this.scheduleInitialConnectionDeadline(connection, connectionId); } catch (error) { if (!this.isCurrentConnection(connection, connectionId)) { @@ -410,11 +419,11 @@ export class ViewerPeer implements ViewerMediaPeer { this.setError(error, "host.fail.connection"); if ( this.recoveryOwner === "viewer" && - this.offerRecoveryAttempts < 1 && + !this.recoveryBudget.exhaustedReported && + this.recoveryBudget.requests < VIEWER_MAX_AUTOMATIC_RECOVERY_REQUESTS && this.events.sendRestartRequest(parentPeerId, connectionId, true) ) { - this.offerRecoveryAttempts += 1; - this.automaticRecoveryRequests = VIEWER_MAX_AUTOMATIC_RECOVERY_REQUESTS; + this.recoveryBudget.requests = VIEWER_MAX_AUTOMATIC_RECOVERY_REQUESTS; this.scheduleRecoveryDeadline(); } else { this.reportRecoveryExhausted(); @@ -479,18 +488,18 @@ export class ViewerPeer implements ViewerMediaPeer { if ( this.disposed || this.recoveryTimer !== null || - this.recoveryExhaustedReported || + this.recoveryBudget.exhaustedReported || this.connection?.connectionState === "connected" || !this.connectionId || !this.parentPeerId ) { return; } - if (this.automaticRecoveryRequests >= VIEWER_MAX_AUTOMATIC_RECOVERY_REQUESTS) { + if (this.recoveryBudget.requests >= VIEWER_MAX_AUTOMATIC_RECOVERY_REQUESTS) { this.reportRecoveryExhausted(); return; } - const rebuild = this.automaticRecoveryRequests > 0; + const rebuild = this.recoveryBudget.requests > 0; if ( !this.events.sendRestartRequest( this.parentPeerId, @@ -501,7 +510,7 @@ export class ViewerPeer implements ViewerMediaPeer { this.reportRecoveryExhausted(); return; } - this.automaticRecoveryRequests += 1; + this.recoveryBudget.requests += 1; this.scheduleRecoveryDeadline(); } @@ -547,7 +556,7 @@ export class ViewerPeer implements ViewerMediaPeer { private reportRecoveryExhausted(): void { if ( - this.recoveryExhaustedReported || + this.recoveryBudget.exhaustedReported || !this.parentPeerId || !this.connectionId ) { @@ -560,7 +569,7 @@ export class ViewerPeer implements ViewerMediaPeer { this.connectionId, ) ?? true; if (reported) { - this.recoveryExhaustedReported = true; + this.recoveryBudget.exhaustedReported = true; } else { this.scheduleRecoveryDeadline(); } @@ -568,8 +577,8 @@ export class ViewerPeer implements ViewerMediaPeer { private resetAutomaticRecovery(): void { this.clearRecoveryTimer(); - this.automaticRecoveryRequests = 0; - this.recoveryExhaustedReported = false; + this.recoveryBudget.requests = 0; + this.recoveryBudget.exhaustedReported = false; } private queueCandidate( diff --git a/tests/native-viewer-peer.test.ts b/tests/native-viewer-peer.test.ts index 105af08a4..89f26a8fc 100644 --- a/tests/native-viewer-peer.test.ts +++ b/tests/native-viewer-peer.test.ts @@ -541,7 +541,7 @@ it.each([ } }); -it.each([false, true])("keeps committed route recovery across backend replacement (replaced=%s)", async (replaced) => { +it.each(["first", "replacement", "failed-offer"])("keeps committed route recovery and its budget across backend replacement (%s)", async (scenario) => { vi.useFakeTimers(); vi.stubGlobal("window", globalThis); vi.stubGlobal("MediaStream", class { getTracks() { return []; } }); @@ -550,8 +550,9 @@ it.each([false, true])("keeps committed route recovery across backend replacemen vi.spyOn(NativeMediaBridge.prototype, "collectMetrics").mockResolvedValue({ ...EMPTY_METRICS }); vi.spyOn(NativeMediaBridge.prototype, "decodedVideoFrames").mockResolvedValue(null); let onEvent!: (event: NativeClientEvent) => void; + const receiveOffer = vi.fn(async () => ({ answer: { type: "answer", sdp: "v=0\r\n" }, audio: false, codec: "vp8" })); const client = { - receiveOffer: async () => ({ answer: { type: "answer", sdp: "v=0\r\n" }, audio: false, codec: "vp8" }), + receiveOffer, onEvent: (fn: typeof onEvent) => { onEvent = fn; return () => undefined; }, onClose: () => () => undefined, closeReceiver: vi.fn(async () => undefined), closeEdge: vi.fn(async () => undefined), @@ -573,13 +574,38 @@ it.each([false, true])("keeps committed route recovery across backend replacemen state("first", "connected"); peer.activatePreparedRoute(); expect(options.recoveryOwner).toBe("route"); // Caller options are not mutable route state. - const connectionId = replaced ? "replacement" : "first"; - if (replaced) { + const connectionId = scenario === "replacement" ? "replacement" : "first"; + if (scenario === "replacement") { await peer.acceptSignal("parent", offer(connectionId)); state(connectionId, "connected"); } state(connectionId, "failed"); expect(restart).toHaveBeenCalledExactlyOnceWith("parent", connectionId, false); expect(exhausted).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(VIEWER_AUTOMATIC_RECOVERY_TIMEOUT_MS); + expect(restart).toHaveBeenNthCalledWith(2, "parent", connectionId, true); + + // A new physical ID is the requested rebuild, not a recovered parent. + const retiredEvent = onEvent; + if (scenario === "failed-offer") receiveOffer.mockRejectedValueOnce(new Error("receiver unavailable")); + await peer.acceptSignal("parent", offer("rebuilt")); + retiredEvent({ version: 9, shareId: "session", connectionId, type: "edge-state", state: "connected" }); + state("rebuilt", "failed"); + expect(restart).toHaveBeenCalledTimes(2); + expect(exhausted).toHaveBeenCalledExactlyOnceWith("parent", "rebuilt"); + + // Explicit retry owns a new bounded request and must report its outcome. + expect(peer.requestRecovery(true)).toBe(true); + await vi.advanceTimersByTimeAsync(VIEWER_AUTOMATIC_RECOVERY_TIMEOUT_MS); + expect(exhausted).toHaveBeenCalledTimes(2); + + // Only actual recovery or a different parent gives a fresh budget. + await peer.acceptSignal("parent", offer("healthy")); + state("healthy", "connected"); + state("healthy", "failed"); + expect(restart).toHaveBeenNthCalledWith(4, "parent", "healthy", false); + await peer.acceptSignal("other-parent", offer("other-connection")); + state("other-connection", "failed"); + expect(restart).toHaveBeenNthCalledWith(5, "other-parent", "other-connection", false); } finally { peer.dispose(); } }); diff --git a/tests/viewer-peer-generation.test.ts b/tests/viewer-peer-generation.test.ts index 0a9d92c33..9f40f2f6e 100644 --- a/tests/viewer-peer-generation.test.ts +++ b/tests/viewer-peer-generation.test.ts @@ -2,8 +2,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { IceConfig, SignalPayload } from "../src/shared/protocol.ts"; import type { PeerSnapshot } from "../src/client/types.ts"; -import { ViewerPeer } from "../src/client/webrtc/viewer-peer.ts"; +import { ViewerPeer, type ViewerPeerEvents } from "../src/client/webrtc/viewer-peer.ts"; import { ViewerSfuRoute } from "../src/client/media/viewer-sfu-route"; +import { NativeCapableViewerPeer } from "../src/client/native/native-viewer-peer"; interface Deferred { promise: Promise; @@ -304,9 +305,9 @@ describe("ViewerPeer connection generations", () => { expect(peer.hasConnectionId(id)).toBe(true); return true; }); - const peer = new ViewerPeer({ iceServers: [] }, { sendSignal, sendRestartRequest, + const peer = new NativeCapableViewerPeer({ iceServers: [] }, { sendSignal, sendRestartRequest, onStream: () => undefined, onUpdate: () => undefined, onRecoveryExhausted: exhausted }, - { recoveryOwner }); + { recoveryOwner }, async () => null, () => undefined, "session", 2); FakePeerConnection.plans.push({ constructorError: new DOMException("Allocation failed", "UnknownError") }); await expect(peer.acceptSignal("host", offer("first"))).resolves.toBeUndefined(); expect(peer.hasConnection()).toBe(false); @@ -724,29 +725,30 @@ describe("ViewerPeer connection generations", () => { expect(timeoutDelays).toEqual(new Map()); }); - it("reports one exhausted edge after bounded restart and rebuild attempts", async () => { + it.each([false, true])("bounds recovery across rebuilt Browser connections (native wrapper=%s)", async (wrapped) => { const restartRequests: Array<{ peerId: string; connectionId: string; rebuild: boolean; }> = []; const exhausted: Array<{ peerId: string; connectionId: string }> = []; - const peer = new ViewerPeer( - { iceServers: [] }, - { - sendSignal: () => true, - sendRestartRequest: (peerId, connectionId, rebuild) => { - restartRequests.push({ peerId, connectionId, rebuild }); - return true; - }, - onStream: () => undefined, - onUpdate: () => undefined, - onRecoveryExhausted: (peerId, connectionId) => { - exhausted.push({ peerId, connectionId }); - return true; - }, + const events: ViewerPeerEvents = { + sendSignal: () => true, + sendRestartRequest: (peerId, connectionId, rebuild) => { + restartRequests.push({ peerId, connectionId, rebuild }); + return true; }, - ); + onStream: () => undefined, + onUpdate: () => undefined, + onRecoveryExhausted: (peerId, connectionId) => { + exhausted.push({ peerId, connectionId }); + return true; + }, + }; + const peer = wrapped + ? new NativeCapableViewerPeer({ iceServers: [] }, events, {}, async () => null, + () => undefined, "session", 2) + : new ViewerPeer({ iceServers: [] }, events); await peer.acceptSignal("relay-parent", offer("failed-connection")); const connection = FakePeerConnection.instances[0]!; connection.connectionState = "failed"; @@ -765,14 +767,25 @@ describe("ViewerPeer connection generations", () => { connectionId: "failed-connection", rebuild: true, }); - timeoutCallbacks.get(3)!(); + await peer.acceptSignal("relay-parent", offer("rebuilt-connection")); + expect(timeoutCallbacks.has(3)).toBe(false); + connection.connectionState = "connected"; + connection.dispatchEvent(new Event("connectionstatechange")); + const rebuilt = FakePeerConnection.instances.at(-1)!; + rebuilt.connectionState = "failed"; + rebuilt.dispatchEvent(new Event("connectionstatechange")); expect(exhausted).toEqual([ - { peerId: "relay-parent", connectionId: "failed-connection" }, + { peerId: "relay-parent", connectionId: "rebuilt-connection" }, ]); - connection.dispatchEvent(new Event("connectionstatechange")); + rebuilt.dispatchEvent(new Event("connectionstatechange")); expect(restartRequests).toHaveLength(2); expect(exhausted).toHaveLength(1); + expect(peer.requestRecovery(true)).toBe(true); + [...timeoutCallbacks.values()].at(-1)!(); + expect(restartRequests).toHaveLength(3); + expect(exhausted).toHaveLength(2); + peer.dispose(); }); it("retries delivery of an exhausted edge after signaling reconnects", async () => { From 80c34d6bb18c26315b07fa2af064b7b230e5739d Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:39:05 +0800 Subject: [PATCH 051/262] docs(transport): record recovery owner and timing limits --- docs/research/nat-traversal.md | 58 ++++++++++++++++++++++------------ docs/todo.md | 4 +-- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 0be016afd..7433b41ff 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1,6 +1,6 @@ # Browser And Native NAT Traversal -Last reviewed: 2026-10-07 +Last reviewed: 2026-10-09 This document owns evidence for improving direct ICE without adding a new relay or a custom transport. The current product contract remains standard @@ -488,26 +488,42 @@ barrier, and probe expiry does not establish media failure or justify replacing healthy media. These controls use fixture SDP exchange; they do not diagnose the product's Trickle ICE or an existing user's failure. -A follow-up kernel fixture on 2026-10-09 matches the committed Viewer's two -three-second recovery opportunities: restart ICE, then rebuild only if still -not connected at the first deadline. It retains tunnel identity and outer NAT -mappings, creates a new inner flow for replacement, and keeps default Pion checks -without an explicit discovery hook. Setup consumes its opportunity; fresh SRTP -is checked separately after connection meets the deadline. Post-restoration -controls cover eight-, nine- and eleven-second blackouts; Tailcat also starts -recovery one or two seconds before restoration. All five Tailcat cases pass two -further race-detector repetitions, recovering through restart or replacement; -the ordinary Pion controls also pass. Failure of the first ICE generation alone -therefore does not establish failure of both existing recovery opportunities. - -The fixture's replacement flow repeats an initial datagram until the SDP exchange -confirms listener acceptance, bounded by that same opportunity. A lone lost -preface had blocked setup before ICE could run. This fixture handshake is not an -accepted product protocol or proof of media readiness. Its complete-gathering -SDP exchange also stalls ordinary Pion when started before network restoration; -that result is not Piik's Trickle ICE behavior. The passing controls narrow the -integration question but do not run the actual Viewer scheduler, room authority -or Browser decoder, and do not establish additional direct reachability. +Manually phased replacement controls give ICE restart and inner-flow replacement +three seconds each, retaining the tunnel, outer mappings and default Pion checks. +Post-restoration controls pass after eight-, nine- and eleven-second blackouts; +two Tailcat controls starting before restoration also pass two further race +repetitions. These windows are stricter than a replacement receiver's actual +15-second initial-connection deadline, not a reproduction of the Viewer schedule. +A bounded repeated fixture preface prevents a lost first UDP packet from blocking +listener acceptance before ICE. It is not product signaling or media-health proof. + +A separate fixture feeds kernel connection states into the actual +`NativeCapableViewerPeer`. Each rebuild gets a new connection ID, as in the +product, while preserving the parent's recovery budget. Renewing that budget +on replacement hid the nine-second outage: three repetitions recovered only +after a third automatic request, outside the two-request contract. The corrected +owner and fixture retain the count across Native and Browser backend replacement. +This independent recovery defect does not establish the self-hosted SFU report's cause. + +With that correction, two-second interruptions recover without restart; +permanent loss exhausts recovery and disposal cancels it. Seven- and eleven-second +Tailcat interruptions pass in the checked run. Nine seconds exhausts the budget: +the replacement sends eight inner Binding checks, receives none, and delivers no +fresh SRTP during its initial deadline or a separate three-second observation +after exhaustion. That observation grants no additional recovery. Native control +and local decoding are stubbed, events are sampled within 100 ms, and SDP waits +for complete gathering. These limits prevent claiming full App integration or +diagnosing a field failure; the composition gap remains an adoption concern. + +The existing [Pion Binding-request setting](https://github.com/pion/webrtc/blob/v4.2.19/settingengine.go#L506) +was then varied without changing the owner, timers or outer tunnel. Both 16- +and 64-check controls initially recovered nine- and eleven-second interruptions. +Six further 16-check cases recovered three times and failed three times during +replacement setup, before its SDP exchange completed. Three narrower reruns with +stage diagnostics passed; the setup failures' precise cause remains unconfirmed. +Increasing inner checks can bridge one discovery phase but has not established +reliable composition. It is not an accepted product setting or reason to add a +second readiness owner, discovery loop or longer deadline. Separate checks preserve continuous sibling audio/video through a local ICE restart, failed-flow retirement, both callers' pending cancellation/deadline, diff --git a/docs/todo.md b/docs/todo.md index 643031529..2d04fa7ed 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -58,8 +58,8 @@ Implementation needs a concrete useful result within the product model. before accepting room-scoped peer keys, flow admission and relay operation. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) - and RTCP/MTU boundaries, plus verification through the actual Viewer recovery - owner and mixed Browser/Native routes. Keep isolated dependency patches and + and RTCP/MTU boundaries, resolution of recovery failures under the actual Viewer + owner, and mixed Browser/Native verification. Keep isolated dependency patches and recovery counterfactuals outside product routing; a diagnostic reply is not media-readiness authority. Research owns the experiments and their limits. No timeout change or transport integration is accepted; the P2P evidence work From 66d2a085e12e3349d39698d32c4d00c93c59b3c9 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:06:33 +0800 Subject: [PATCH 052/262] docs(transport): record listener isolation boundary --- docs/research/nat-traversal.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 7433b41ff..21612fc68 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -376,6 +376,16 @@ Pion and Tailcat, including TWCC and keyframe requests. Tailcat recovered an injected missing video packet through existing NACK handling after DERP closed. This is forwarding evidence, not throughput or room-controller acceptance. +A 2026-10-09 inbound-only control found that opening or closing an unrelated +Tailcat listener discards an accepted UDP flow's filter state. The unchanged +control passes; both listener changes stop delivery. Continuous outbound media +can recreate the reverse-flow entry, explaining why bidirectional checks alone +miss this boundary. The isolated repair passes the previous filter to Tailscale's +[existing shared-state API](https://github.com/tailscale/tailscale/blob/a0e471a35b8f/wgengine/filter/filter.go#L218). +Three race repetitions, the copied Tailcat listener suite and the combined +revocation/isolation controls pass. The patch remains experimental; it neither +changes Piik nor fixes the independent inner-ICE recovery failure below. + A separate real Chrome control decoded H.264/Opus and VP8/Opus from a generated 640×360 picture and tone through Browser WebRTC → Pion/Tailcat/Pion → Browser WebRTC without re-encoding. Picture and decoded audio continued after DERP @@ -507,7 +517,7 @@ This independent recovery defect does not establish the self-hosted SFU report's With that correction, two-second interruptions recover without restart; permanent loss exhausts recovery and disposal cancels it. Seven- and eleven-second -Tailcat interruptions pass in the checked run. Nine seconds exhausts the budget: +Tailcat interruptions pass in the checked run. Nine seconds can exhaust the budget: the replacement sends eight inner Binding checks, receives none, and delivers no fresh SRTP during its initial deadline or a separate three-second observation after exhaustion. That observation grants no additional recovery. Native control @@ -521,7 +531,11 @@ and 64-check controls initially recovered nine- and eleven-second interruptions. Six further 16-check cases recovered three times and failed three times during replacement setup, before its SDP exchange completed. Three narrower reruns with stage diagnostics passed; the setup failures' precise cause remains unconfirmed. -Increasing inner checks can bridge one discovery phase but has not established +With the separate listener-state repair, paired counters still reproduce a +default-budget failure: the server receives the client's inner STUN packets, +while the client receives none. Four additional 16-check controls recover; +they do not resolve the earlier setup failures. Increasing inner checks can +bridge one discovery phase but has not established reliable composition. It is not an accepted product setting or reason to add a second readiness owner, discovery loop or longer deadline. From 90d005f06f88b6a5217e1a3bc8199b2011bfc57b Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:44:04 +0800 Subject: [PATCH 053/262] fix(routing): preserve new-session recovery opportunities --- docs/research/nat-traversal.md | 1 + docs/standards/routing-transport.md | 3 + docs/todo.md | 11 ++- internal/server/route/api.go | 2 + internal/server/route/controller.go | 3 + internal/server/route/harness_test.go | 3 + internal/server/route/operation.go | 2 + internal/server/route/opportunities.go | 22 ++--- .../server/route/session_opportunity_test.go | 64 +++++++++++++ internal/server/signal/router_failure_test.go | 91 +++++++++++++++++++ scripts/embedded-sfu-gate.ts | 12 ++- scripts/embedded-sfu-page.ts | 15 ++- 12 files changed, 211 insertions(+), 18 deletions(-) create mode 100644 internal/server/route/session_opportunity_test.go diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 21612fc68..de723193f 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -488,6 +488,7 @@ disconnect-grace, restart and rebuild schedule. | One probe on each side before default-Pion restart | Nine- and eleven-second blackouts recovered in three race-detector repetitions, including both probes and SDP exchange inside three seconds. This requires an isolated, unsupported Server hook. | | Same bilateral probe begins while the network is still down | After eight seconds of blackout, restoration one or two seconds into the probe still left it waiting until its three-second deadline in repeated race checks. Post-restoration controls continued to pass. | | Lost bilateral probes, retain the existing media connection | In six race checks, both probes expired while WebRTC was already connected; a subsequent observation received fresh SRTP without new SDP, sockets or credentials. The separate three-second observation is not a product recovery extension. | +| New flow confirms encrypted UDP in both directions before inner ICE | With the listener-state repair and actual Viewer owner, one of four nine-/eleven-second controls recovered; three timed out in the existing three-second preparation, before SDP. The bounded fixture echo changes where waiting occurs, not the underlying recovery limit. | These controls limit the bilateral proposal: a [`DiscoPing` call](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/tailcat.go#L2258) diff --git a/docs/standards/routing-transport.md b/docs/standards/routing-transport.md index a2a97ac6f..4397c54b9 100644 --- a/docs/standards/routing-transport.md +++ b/docs/standards/routing-transport.md @@ -217,6 +217,9 @@ An active configuration describes that connection; it cannot create a replacemen Browser subscriber. Replacement needs the controller's prepared subscription and first-frame proof. Reauthentication during an unfinished restart also returns that subscription to its route owner; sending an answer is not connection proof. +Retained media keeps its original acquisition opportunity when signaling sessions +change. Its failure cannot consume a new session's untried opportunity; a new +physical connection binds the current one. This applies to Peer and SFU edges. An exhausted opportunity remains an explicit failure until an eligible new fact reopens acquisition. Source capture remains Host-owned throughout. diff --git a/docs/todo.md b/docs/todo.md index 2d04fa7ed..a65e471d3 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -15,9 +15,6 @@ Git/PRs own completed history. coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. - Windows full Go acceptance needs a runner that permits its test executables; - the current run stopped before the Browser-launcher package tests with an OS - access denial. Separate native checks and Linux checks do not close that gap. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance are complete. Integrate through the next owner-approved publishing merge and verify the separate workflow with its original CI artifacts. Mirror failure @@ -130,6 +127,14 @@ of a reporter's cause; research owns the completed experiments and their limits. The local Native H264/SFU-only control passes, but does not reproduce the reporter's network. Correlate transport-state diagnostics with the affected Viewer before changing admission or recovery. + [#468](https://github.com/TNTcraftHIM/Piik/issues/468) needs the complete join + trace to distinguish a new Viewer from reauthentication with lost local media; + its excerpt starts with an already committed SFU edge. Retest the affected + deployment with the repaired retained-connection opportunity accounting. + [#467](https://github.com/TNTcraftHIM/Piik/issues/467)'s Viewer report reaches + four preparations and exhaustion in about 170 ms, without local peer creation + or a Viewer failure report. Obtain the paired Host/App rejection before + attributing this attempt to NAT or extending its network timeout. [#465](https://github.com/TNTcraftHIM/Piik/issues/465) reports Chrome/Firefox LAN asymmetry, local-pool timeout and separate public-path quality loss. Its successful repeat changed both local-network permission and IP masking. diff --git a/internal/server/route/api.go b/internal/server/route/api.go index 1311e950d..bb9633e80 100644 --- a/internal/server/route/api.go +++ b/internal/server/route/api.go @@ -224,6 +224,8 @@ type CommittedEdge struct { Usable bool PhysicalActive bool Resource *Resource + // The acquisition's session key stays fixed when signaling ownership rebinds. + opportunityKey string } // HostPublication is the single live SFU ingress. diff --git a/internal/server/route/controller.go b/internal/server/route/controller.go index 221b902be..a180f07d6 100644 --- a/internal/server/route/controller.go +++ b/internal/server/route/controller.go @@ -742,6 +742,9 @@ func (c *Controller) AdoptDirectConnection(input AdoptDirectConnectionInput, now } c.clearQualityForParticipant(input.ChildPeerID) edge.ConnectionID = input.NewConnectionID + edge.opportunityKey = c.candidateOpportunityBase(input.ChildPeerID, CandidatePlan{ + Tuple: CandidateTuple{Kind: edge.Kind, ParentPeerID: edge.ParentPeerID}, + }) if child, ok := c.participants.Get(input.ChildPeerID); ok { child.availabilityExhausted = false } diff --git a/internal/server/route/harness_test.go b/internal/server/route/harness_test.go index 82261b1ff..0041a5565 100644 --- a/internal/server/route/harness_test.go +++ b/internal/server/route/harness_test.go @@ -242,6 +242,9 @@ func (c *Controller) hydrateEdge(childPeerID string, edge committedEdgeSeed) { Usable: edge.Usable, PhysicalActive: edge.PhysicalActive, Resource: edge.Resource, + opportunityKey: c.candidateOpportunityBase(childPeerID, CandidatePlan{ + Tuple: CandidateTuple{Kind: edge.Kind, ParentPeerID: edge.ParentPeerID}, + }), } if edge.Kind == UpstreamPeer { if parent, ok := c.participants.Get(edge.ParentPeerID); ok { diff --git a/internal/server/route/operation.go b/internal/server/route/operation.go index 10bc58fb0..ce9742570 100644 --- a/internal/server/route/operation.go +++ b/internal/server/route/operation.go @@ -948,6 +948,7 @@ func (c *Controller) commitAttempt(op *operation, att *attempt) []*Resource { ConnectionID: att.connectionID, Usable: true, PhysicalActive: true, + opportunityKey: c.candidateOpportunityBase(op.childPeerID, CandidatePlan{Tuple: att.tuple}), }) c.directContinuations.Delete(op.childPeerID) } else { @@ -981,6 +982,7 @@ func (c *Controller) commitAttempt(op *operation, att *attempt) []*Resource { Usable: true, PhysicalActive: true, Resource: att.reservation.Edge, + opportunityKey: c.candidateOpportunityBase(op.childPeerID, CandidatePlan{Tuple: att.tuple}), }) var remainingParentPeerIDs []string if op.reason == DemandSfuBootstrap && oldPeer && old.Usable && old.PhysicalActive { diff --git a/internal/server/route/opportunities.go b/internal/server/route/opportunities.go index 8407d805e..96e2645db 100644 --- a/internal/server/route/opportunities.go +++ b/internal/server/route/opportunities.go @@ -165,19 +165,19 @@ func (c *Controller) consumeQualityOperationCandidates(op *operation) { // consumeActiveEdgeOpportunity exhausts the invalidated edge's exact tuple at rank 0. func (c *Controller) consumeActiveEdgeOpportunity(child *participant, edge *CommittedEdge) { - if edge.Kind == UpstreamPeer { - key := "peer:" + edge.ParentPeerID + "\x00" + child.sessionID + "\x00" + edge.ParentSessionID - previous, _ := child.consumedCandidateOpportunities.Get(key) - child.consumedCandidateOpportunities.Set(key, candidateOpportunity{ - rank: 0, - startedAttempts: previous.startedAttempts, - exhausted: true, - }) + key := c.candidateOpportunityBase(child.peerID, CandidatePlan{ + Tuple: CandidateTuple{Kind: edge.Kind, ParentPeerID: edge.ParentPeerID}, + }) + // A retained connection can fail after reauthentication. Its old acquisition + // must not exhaust the replacement session's still-untried opportunity. + if edge.opportunityKey != key { return } - c.consumeCandidateOpportunity(child.peerID, CandidatePlan{ - Tuple: CandidateTuple{Kind: UpstreamSfu, Publication: PublicationReuse}, - EndpointTransition: EndpointTransition{Kind: TransitionNone}, + previous, _ := child.consumedCandidateOpportunities.Get(key) + child.consumedCandidateOpportunities.Set(key, candidateOpportunity{ + rank: 0, + startedAttempts: previous.startedAttempts, + exhausted: true, }) } diff --git a/internal/server/route/session_opportunity_test.go b/internal/server/route/session_opportunity_test.go new file mode 100644 index 000000000..4450fdc8a --- /dev/null +++ b/internal/server/route/session_opportunity_test.go @@ -0,0 +1,64 @@ +package route + +import "testing" + +func TestRetainedEdgeFailurePreservesNewSessionOpportunity(t *testing.T) { + for _, upstream := range []UpstreamKind{UpstreamPeer, UpstreamSfu} { + for _, changed := range []string{"none", "child", "host"} { + t.Run(string(upstream)+"/"+changed, func(t *testing.T) { + routes := newController(2, Options{SfuEnabled: true}) + addViewer(routes, A, 0, nil) + if upstream == UpstreamSfu { + routes.hydrateHostPublication("publication", res("publication_resource"), "") + routes.hydrateEdge(A, sfuEdge("publication", "retained_connection", "subscription")) + } else { + routes.hydrateEdge(A, peerEdge(HOST, "retained_connection")) + } + switch changed { + case "child": + routes.UpsertParticipant(viewerInput(A, "replacement_child_session", 0), nil) + case "host": + routes.UpsertParticipant(hostInput("replacement_host_session", 2), nil) + } + // Reauthentication keeps healthy media and its physical identity. + edge := edgeOf(t, routes, A) + eq(t, edge.Usable, true) + eq(t, edge.ConnectionID, "retained_connection") + noOperation(t, routes.Reconcile(0).Operation) + eq(t, routes.InvalidateEdge(EdgeGuard{ + ChildPeerID: A, ChildSessionID: edge.ChildSessionID, + ParentSessionID: edge.ParentSessionID, + RouteRevision: routes.Snapshot().Revision, ConnectionID: edge.ConnectionID, + }, ms(1)), true) + operation := must(t, routes.Reconcile(1).Operation) + eq(t, hasTuple(operation, func(tuple CandidateTuple) bool { + return tuple.Kind == upstream + }), changed != "none") + }) + } + } +} + +func TestRebuiltDirectConnectionConsumesCurrentSessionOpportunity(t *testing.T) { + routes := newController(2, Options{SfuEnabled: true}) + addViewer(routes, A, 0, ms(0)) + routes.Reconcile(0) + commitCurrent(t, routes, 1, "original_connection") + routes.UpsertParticipant(viewerInput(A, "replacement_child_session", 0), nil) + edge := edgeOf(t, routes, A) + result := routes.AdoptDirectConnection(AdoptDirectConnectionInput{ + EdgeGuard: EdgeGuard{ + ChildPeerID: A, ChildSessionID: edge.ChildSessionID, + ParentSessionID: edge.ParentSessionID, RouteRevision: routes.Snapshot().Revision, + ConnectionID: edge.ConnectionID, + }, + NewConnectionID: "rebuilt_connection", + }, 3) + eq(t, result.Accepted, true) + eq(t, routes.InvalidateEdge(EdgeGuard{ + ChildPeerID: A, ChildSessionID: edge.ChildSessionID, + ParentSessionID: edge.ParentSessionID, RouteRevision: result.ActiveRevision, + ConnectionID: "rebuilt_connection", + }, ms(4)), true) + eq(t, hasTuple(must(t, routes.Reconcile(4).Operation), isPeer), false) +} diff --git a/internal/server/signal/router_failure_test.go b/internal/server/signal/router_failure_test.go index cc092abfd..000ce5705 100644 --- a/internal/server/signal/router_failure_test.go +++ b/internal/server/signal/router_failure_test.go @@ -4,8 +4,99 @@ import ( "testing" "github.com/TNTcraftHIM/Piik/internal/server/protocol" + "github.com/TNTcraftHIM/Piik/internal/server/room" + "github.com/TNTcraftHIM/Piik/internal/server/sfu" ) +func TestRouterPreparesSfuForNewAndReauthenticatedViewers(t *testing.T) { + for _, reauthenticate := range []bool{false, true} { + name := "new-viewer" + if reauthenticate { + name = "lost-retained-media" + } + t.Run(name, func(t *testing.T) { + h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true, fakeTimers: true}) + created := h.createRoom() + _, first, sibling := h.establishSfuRoom(created) + viewer := first + if reauthenticate { + viewer.sessionID = "replacement_viewer_session_12345678" + h.locked(func() { + connected, err := h.store.ConnectParticipant(room.ConnectParticipantInput{ + RoomID: created.RoomID, Role: protocol.RoleViewer, + ViewerGrant: created.ViewerGrant, + ClientID: "viewer_client_sfu-root-" + created.RoomID + "_12345678", + SessionID: viewer.sessionID, + }) + if err != nil || connected.PeerID != first.peerID { + t.Fatalf("viewer replacement changed identity: %v", err) + } + }) + } else { + viewer = h.connectViewer(created, "late-viewer") + } + var retained activeViewerMediaEdge + h.locked(func() { + if reauthenticate { + retained = h.doMustEdge(created.RoomID, viewer.peerID) + } + state := h.doComplete(viewer) + expected := protocol.NoUpstream() + if reauthenticate { + expected = protocol.SfuUpstream() + } + if state.routeAssignment.Upstream != expected { + t.Fatalf("authentication upstream = %v, want %v", state.routeAssignment.Upstream, expected) + } + }) + if reauthenticate { + h.settle() + if sfuConfigCount(h.messages(viewer.sessionID)) != 1 { + t.Fatal("healthy retained media must receive its current configuration") + } + for _, message := range h.messages(viewer.sessionID) { + if _, ok := message.(protocol.RouteUpdatePrepareMessage); ok { + t.Fatal("reauthentication alone must not replace healthy media") + } + } + h.routeFailed(viewer, routeFailedMessage(retained.revision, "active", retained.connectionID)) + } + prepared := h.waitPreparedTransport(viewer.sessionID, "sfu") + if prepared.Candidate.ConnectionID == retained.connectionID { + t.Fatal("replacement must allocate a new physical connection") + } + preparedSeen, configured := false, false + for _, message := range h.messages(viewer.sessionID) { + switch value := message.(type) { + case protocol.RouteUpdatePrepareMessage: + preparedSeen = value.Candidate.ConnectionID == prepared.Candidate.ConnectionID + case protocol.SfuConfigMessage: + if value.ConnectionID == prepared.Candidate.ConnectionID { + if !preparedSeen { + t.Fatal("new subscription configuration preceded its preparation") + } + configured = true + } + } + } + if !configured { + t.Fatal("prepared subscription has no configuration") + } + h.routeReady(viewer, int64(prepared.Revision)) + h.locked(func() { + if h.doMustEdge(created.RoomID, sibling.peerID).upstream != protocol.SfuUpstream() { + t.Fatal("healthy sibling lost its SFU route") + } + }) + egress := 3 + if reauthenticate { + egress = 2 + } + h.waitUsage(sfu.Usage{Ingress: 1, Egress: egress}) + }) + } +} + func TestRouterFencesDelayedActiveFailureByPhysicalConnection(t *testing.T) { for _, transport := range []string{"peer", "sfu"} { for _, role := range []string{"host", "viewer"} { diff --git a/scripts/embedded-sfu-gate.ts b/scripts/embedded-sfu-gate.ts index 47b7c5b4f..38d1bb78c 100644 --- a/scripts/embedded-sfu-gate.ts +++ b/scripts/embedded-sfu-gate.ts @@ -105,7 +105,7 @@ async function main(): Promise { displayLifecycle: null as { hiddenFrames: number; hiddenElapsedMs: number; lowLayerDecoded: boolean; resumed: boolean; replaced: boolean } | null, mediaDiagnostics: null as unknown, sourceDrawing: null as { draws: number; elapsedMs: number; visibility: string } | null, - publicationRetired: false, subscriptionRecovered: false, nativeShareStopped: !nativeArm, udpReleased: false, cleanup: false, error: null as string | null }; + publicationRetired: false, subscriptionRecovered: false, pageReloadRecovered: false, nativeShareStopped: !nativeArm, udpReleased: false, cleanup: false, error: null as string | null }; let server: ChildProcessWithoutNullStreams | null = null; let chrome: ChildProcessWithoutNullStreams | null = null; let vite: ViteDevServer | null = null; @@ -315,6 +315,14 @@ async function main(): Promise { result.low = await call(viewer, "gate.waitForFrames(854, 480)"); await waitForSample(() => call(viewer!, "gate.audioEnergy()", 3_000), (energy) => { result.audioEnergy.low = energy; return energy > result.audioEnergy.high; }, 5_000); + result.stage = "viewer-media-reload"; + const retainedSubscription = (await call(viewer, "gate.snapshot()")).connectionId; + // A reload loses local media before reporting failure, while room authority + // still retains the old committed edge inside the reconnect grace period. + await call(viewer, `gate.reconnectViewer(${JSON.stringify(room)}, true)`); + await until(viewer, value => value.committed && value.connectionId !== retainedSubscription && value.audioKbps > 0); + await call(viewer, "gate.waitForFrames(854, 480)"); + result.pageReloadRecovered = true; result.stage = "committed-subscription-recovery"; const previousSubscription = await call(viewer, "gate.failSubscription()"); // Terminal failure consumes this exact opportunity. A new authenticated @@ -384,7 +392,7 @@ async function main(): Promise { (nativeArm || result.stereoSeparationDb?.every(db => db > 20) === true) && result.viewer.audioKbps > 0 && result.audioEnergy.high > 0 && result.audioEnergy.low > result.audioEnergy.high && result.high !== null && result.low !== null && result.audioEnergy.recovered > 0 && - result.publicationRetired && result.subscriptionRecovered && result.cleanup && result.nativeShareStopped && + result.publicationRetired && result.subscriptionRecovered && result.pageReloadRecovered && result.cleanup && result.nativeShareStopped && (!displayArm || result.displayLifecycle?.hiddenFrames === 150 && result.displayLifecycle.lowLayerDecoded && result.displayLifecycle.resumed && result.displayLifecycle.replaced) && (!result.secondRoom || result.secondRoom.connected && result.secondRoom.framesAfterFirstStopped >= 30 && diff --git a/scripts/embedded-sfu-page.ts b/scripts/embedded-sfu-page.ts index a6d588119..43f3a2060 100644 --- a/scripts/embedded-sfu-page.ts +++ b/scripts/embedded-sfu-page.ts @@ -157,7 +157,9 @@ async function accept(message: ServerMessage, host: boolean): Promise { } } if (message.type === "sfu-config") { - configuration = message; + // A late refresh can describe the old edge after a newer prepare. The route + // owner rejects it; the fixture's reported identity must not move backward. + if (!configuration || message.revision >= configuration.revision) configuration = message; if (host) { await hostRoute?.acceptConfig(message); published = publisher !== null; @@ -173,6 +175,7 @@ async function connect(room: CreateRoomResponse, host: boolean): Promise { socket = new WebSocket(new URL("/signal", location.href).href.replace(/^http/, "ws")); const connection = socket; socket.addEventListener("message", ({ data }) => { + if (socket !== connection) return; messageTail = messageTail.then(() => accept(decodeServerMessage(String(data)), host)).catch(failed); }); socket.addEventListener("error", () => failed("Room WebSocket failed")); @@ -295,10 +298,18 @@ export async function startViewer(room: CreateRoomResponse): Promise { await connect(room, false); } -export async function reconnectViewer(room: CreateRoomResponse): Promise { +export async function reconnectViewer(room: CreateRoomResponse, discardMedia = false): Promise { const previous = socket; socket = null; previous?.close(); + if (discardMedia) { + await messageTail; + await viewerRoute?.disconnect(); + viewerRoute = null; + video?.remove(); + video = null; + decoded = committed = false; + } authenticated = false; await connect(room, false); } From 11005952aee3271280b3b6035e8e7c4cb6eaea73 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:04:54 +0800 Subject: [PATCH 054/262] docs(network): qualify transport recovery evidence --- docs/research/nat-traversal.md | 16 ++++++++++++++++ docs/todo.md | 3 ++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index de723193f..34720f9d6 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -526,6 +526,22 @@ and local decoding are stubbed, events are sampled within 100 ms, and SDP waits for complete gathering. These limits prevent claiming full App integration or diagnosing a field failure; the composition gap remains an adoption concern. +Matched plain-Pion controls through that same owner recover two- and nine-second +outages in two repetitions. Both eleven-second controls instead reach the +fixture's three-second complete-gathering limit: the ICE restart begins while +the network is still unavailable. Piik uses Trickle ICE, so this is a fixture +limit, not an established product failure or Tailcat advantage. Full comparative +recovery claims require matching that signaling behavior first. + +Bounded anonymous traces further localize two Tailcat encrypted-echo failures. +The replacement flow is accepted and its client preface read at the server, +but the reverse read expires after three seconds before SDP. The server selects +its direct path near the end of that window; the client has selected its path +about two seconds earlier. No filter-drop event was observed, which is not +packet-delivery proof. Other timing phases recover. This supports a directional +underlay scheduling limit; it does not establish a new authorization defect, +a safe readiness barrier, or a reason to widen product deadlines. + The existing [Pion Binding-request setting](https://github.com/pion/webrtc/blob/v4.2.19/settingengine.go#L506) was then varied without changing the owner, timers or outer tunnel. Both 16- and 64-check controls initially recovered nine- and eleven-second interruptions. diff --git a/docs/todo.md b/docs/todo.md index a65e471d3..467ee7f60 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -56,7 +56,8 @@ Implementation needs a concrete useful result within the product model. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) and RTCP/MTU boundaries, resolution of recovery failures under the actual Viewer - owner, and mixed Browser/Native verification. Keep isolated dependency patches and + owner with Trickle ICE rather than the fixture's complete-gathering wait, and + mixed Browser/Native verification. Keep isolated dependency patches and recovery counterfactuals outside product routing; a diagnostic reply is not media-readiness authority. Research owns the experiments and their limits. No timeout change or transport integration is accepted; the P2P evidence work From 7d20e477e78abc046092772df7aa14d3abfa0c1b Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:30:06 +0800 Subject: [PATCH 055/262] docs(transport): record matched trickle recovery boundary --- docs/research/nat-traversal.md | 34 ++++++++++++++++++---------------- docs/todo.md | 6 +++--- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 34720f9d6..f9925d254 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -516,22 +516,24 @@ after a third automatic request, outside the two-request contract. The corrected owner and fixture retain the count across Native and Browser backend replacement. This independent recovery defect does not establish the self-hosted SFU report's cause. -With that correction, two-second interruptions recover without restart; -permanent loss exhausts recovery and disposal cancels it. Seven- and eleven-second -Tailcat interruptions pass in the checked run. Nine seconds can exhaust the budget: -the replacement sends eight inner Binding checks, receives none, and delivers no -fresh SRTP during its initial deadline or a separate three-second observation -after exhaustion. That observation grants no additional recovery. Native control -and local decoding are stubbed, events are sampled within 100 ms, and SDP waits -for complete gathering. These limits prevent claiming full App integration or -diagnosing a field failure; the composition gap remains an adoption concern. - -Matched plain-Pion controls through that same owner recover two- and nine-second -outages in two repetitions. Both eleven-second controls instead reach the -fixture's three-second complete-gathering limit: the ICE restart begins while -the network is still unavailable. Piik uses Trickle ICE, so this is a fixture -limit, not an established product failure or Tailcat advantage. Full comparative -recovery claims require matching that signaling behavior first. +The actual-owner fixture now supports incremental SDP/candidate exchange, +matching Piik's [Trickle ICE generation boundary](https://www.rfc-editor.org/rfc/rfc8838.html#section-13). +With this enabled, ordinary Pion recovers two-, nine- and eleven-second outages +in three runs each. Without it, the same eleven-second control still fails at +the fixture's three-second gathering barrier while the network is unavailable. +That negative control is a fixture limitation, not a product defect. + +In the matched Trickle run with the isolated listener-state repair, Tailcat +recovers two- and seven-second interruptions. Nine- and eleven-second outages +exhaust recovery: the replacement sends eight inner Binding checks, receives +none, and delivers no fresh SRTP during its initial deadline or a separate +three-second observation after exhaustion. Both 16-check cases pass; one +64-check replacement instead fails before SDP while awaiting its inner flow. +Permanent loss exhausts the existing budget and disposal cancels recovery. +No observation grants another attempt or extends a deadline. Native control +and local decoding remain stubs, and events are sampled within 100 ms. These +limits prevent claiming full App integration or diagnosing a field failure; +the composition gap remains an adoption concern. Bounded anonymous traces further localize two Tailcat encrypted-echo failures. The replacement flow is accepted and its client preface read at the server, diff --git a/docs/todo.md b/docs/todo.md index 467ee7f60..94b628d15 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -55,9 +55,9 @@ Implementation needs a concrete useful result within the product model. before accepting room-scoped peer keys, flow admission and relay operation. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) - and RTCP/MTU boundaries, resolution of recovery failures under the actual Viewer - owner with Trickle ICE rather than the fixture's complete-gathering wait, and - mixed Browser/Native verification. Keep isolated dependency patches and + and RTCP/MTU boundaries, resolution of the reproduced recovery failures under + the actual Viewer owner with Trickle ICE, and mixed Browser/Native verification. + Keep isolated dependency patches and recovery counterfactuals outside product routing; a diagnostic reply is not media-readiness authority. Research owns the experiments and their limits. No timeout change or transport integration is accepted; the P2P evidence work From c1c9c18bfbc87a17ee38c7520c3ea82a1718c93e Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:52:20 +0800 Subject: [PATCH 056/262] docs(transport): qualify retained-flow recovery candidate --- docs/research/nat-traversal.md | 33 ++++++++++++++++++++------------- docs/todo.md | 5 +++-- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index f9925d254..2dd58eccb 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -544,19 +544,26 @@ packet-delivery proof. Other timing phases recover. This supports a directional underlay scheduling limit; it does not establish a new authorization defect, a safe readiness barrier, or a reason to widen product deadlines. -The existing [Pion Binding-request setting](https://github.com/pion/webrtc/blob/v4.2.19/settingengine.go#L506) -was then varied without changing the owner, timers or outer tunnel. Both 16- -and 64-check controls initially recovered nine- and eleven-second interruptions. -Six further 16-check cases recovered three times and failed three times during -replacement setup, before its SDP exchange completed. Three narrower reruns with -stage diagnostics passed; the setup failures' precise cause remains unconfirmed. -With the separate listener-state repair, paired counters still reproduce a -default-budget failure: the server receives the client's inner STUN packets, -while the client receives none. Four additional 16-check controls recover; -they do not resolve the earlier setup failures. Increasing inner checks can -bridge one discovery phase but has not established -reliable composition. It is not an accepted product setting or reason to add a -second readiness owner, discovery loop or longer deadline. +Varying the existing [Pion Binding-request setting](https://github.com/pion/webrtc/blob/v4.2.19/settingengine.go#L506) +can bridge a discovery delay, but repeated controls also fail before SDP during +new-flow setup. Paired counters separately show the server receiving inner STUN +while the client receives none. More checks cannot repair every setup failure; +neither a new readiness owner nor a longer product deadline is accepted. + +A retained-flow counterfactual reuses Pion's existing +[UDP mux](https://github.com/pion/ice/blob/v4.4.0/udp_mux.go) after closing the old +PeerConnection. It keeps the accepted Tailcat flow and replaces ICE/DTLS/SRTP, +without another listener or preparation exchange. Setup falls from about 913 ms +to 11–20 ms, but only three of six default-check nine-/eleven-second controls +recover. The other three exhaust inner checks without fresh SRTP. This removes +flow creation from those failures; it does not by itself resolve recovery. +Combining retained flow with 16 checks still fails two of six cases. With 64 +checks, all six nine-/eleven-second cases recover; permanent loss still exhausts +the existing two-request budget after the replacement's 15-second initial +deadline, and disposal cancels recovery. This identifies a bounded composition +candidate without another timer or readiness probe. The fixture remains serial: +it does not establish safe overlapping connections, a room-authorized retained-flow +owner, full App interoperability or any additional direct-reachability benefit. Separate checks preserve continuous sibling audio/video through a local ICE restart, failed-flow retirement, both callers' pending cancellation/deadline, diff --git a/docs/todo.md b/docs/todo.md index 94b628d15..d7f212368 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -55,8 +55,9 @@ Implementation needs a concrete useful result within the product model. before accepting room-scoped peer keys, flow admission and relay operation. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) - and RTCP/MTU boundaries, resolution of the reproduced recovery failures under - the actual Viewer owner with Trickle ICE, and mixed Browser/Native verification. + and RTCP/MTU boundaries, retained-flow recovery beyond the passing serial + Trickle/64-check experiment, and mixed Browser/Native verification. Default + inner checks still fail; no new product setting or flow owner is accepted. Keep isolated dependency patches and recovery counterfactuals outside product routing; a diagnostic reply is not media-readiness authority. Research owns the experiments and their limits. From 03c80ff3a824632381118260643aad92db27fd66 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:01:21 +0800 Subject: [PATCH 057/262] docs(transport): record retained-flow isolation evidence --- docs/research/nat-traversal.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 2dd58eccb..60ebeda2a 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -561,9 +561,12 @@ Combining retained flow with 16 checks still fails two of six cases. With 64 checks, all six nine-/eleven-second cases recover; permanent loss still exhausts the existing two-request budget after the replacement's 15-second initial deadline, and disposal cancels recovery. This identifies a bounded composition -candidate without another timer or readiness probe. The fixture remains serial: -it does not establish safe overlapping connections, a room-authorized retained-flow -owner, full App interoperability or any additional direct-reachability benefit. +candidate without another timer or readiness probe. Each retained flow is serial: +overlapping connections still use separate flows. Three local race controls keep +sibling audio/video flowing through two retained-flow replacements and another +flow's rollback; peer revocation stops fresh media on both remaining flows. +These checks do not define room-grant integration or establish full App +interoperability or any additional direct-reachability benefit. Separate checks preserve continuous sibling audio/video through a local ICE restart, failed-flow retirement, both callers' pending cancellation/deadline, From 9b67045a3833c4f12a36ec38291af7fd7bb29241 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:14:28 +0800 Subject: [PATCH 058/262] docs(network): qualify SFU forwarding behind NAT --- docs/research/nat-traversal.md | 17 +++++++++++++++++ docs/todo.md | 6 ++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 60ebeda2a..4815d956c 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -27,6 +27,23 @@ configured, Chromium emitted one deduplicated srflx candidate. No raw address or port was retained. This proves endpoint reachability and candidate deduplication; it does not classify the NAT or prove another direct path. +### SFU Behind Server NAT + +An isolated Linux kernel-NAT control on 2026-10-09 places the current Server +behind same-port UDP forwarding with `SFU_PUBLIC_IP`; direct access to its private +address is blocked. Chromium 152 Host/Viewer pages use the real SFU route owners +and authenticated signaling under `SFU_ONLY`. VP8/Opus reception, 720p-to-480p +changes, fresh decoding after Viewer reauthentication and retirement all pass. + +Blocking only the Viewer process's UDP preserves connected Host ingress and +LiveKit's `downtrack bound`, but the Viewer remains connecting without RTP or +decoded frames. Its transport fails after about 15 seconds and the controller +rejects that candidate without committing a publication. Thus binding plus +healthy ingress cannot establish Viewer readiness or a controller defect. +This does not reproduce the reported Native/phone attempt or its 20-second +deadline: matched Viewer transport/decode evidence is still required. The control +changes no product deadline, retry or first-frame rule. + ## Browser Prediction Preflight A 2026-09-01 Chrome 151 preflight repeated the combined self-hosted plus diff --git a/docs/todo.md b/docs/todo.md index d7f212368..e437c7331 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -126,8 +126,10 @@ of a reporter's cause; research owns the completed experiments and their limits. `SFU_ONLY`. They still lack the matched Viewer ICE, decode and `route-ready` trace. The cited Host failure follows the operation deadline; healthy ingress and a bound subscription do not establish delivered frames or its cause. - The local Native H264/SFU-only control passes, but does not reproduce the - reporter's network. Correlate transport-state diagnostics with the affected + Local Native H264 and [kernel-NAT Browser SFU controls](./research/nat-traversal.md#sfu-behind-server-nat) + pass, but do not reproduce the reporter's network. The blocked-Viewer control + preserves healthy ingress and a bound track without usable Viewer media. + Correlate transport-state diagnostics with the affected Viewer before changing admission or recovery. [#468](https://github.com/TNTcraftHIM/Piik/issues/468) needs the complete join trace to distinguish a new Viewer from reauthentication with lost local media; From 86e53aabd8e35afc965dda48d63d5f90be56cf2e Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:40:06 +0800 Subject: [PATCH 059/262] docs(network): verify native SFU and media-loss boundaries --- docs/research/nat-traversal.md | 16 +++++++++++++--- docs/todo.md | 7 ++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 4815d956c..1331fa0d2 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -34,15 +34,22 @@ behind same-port UDP forwarding with `SFU_PUBLIC_IP`; direct access to its priva address is blocked. Chromium 152 Host/Viewer pages use the real SFU route owners and authenticated signaling under `SFU_ONLY`. VP8/Opus reception, 720p-to-480p changes, fresh decoding after Viewer reauthentication and retirement all pass. +Windows Native H264/Opus also passes through QEMU UDP forwarding and the Linux +NAT, decoding 300 720p frames in Chrome 154, changing to 480p, recovering its +subscription and reauthenticating after local media loss. The App captures a +dedicated generated window; this does not reproduce a phone or carrier network. Blocking only the Viewer process's UDP preserves connected Host ingress and LiveKit's `downtrack bound`, but the Viewer remains connecting without RTP or decoded frames. Its transport fails after about 15 seconds and the controller rejects that candidate without committing a publication. Thus binding plus healthy ingress cannot establish Viewer readiness or a controller defect. -This does not reproduce the reported Native/phone attempt or its 20-second -deadline: matched Viewer transport/decode evidence is still required. The control -changes no product deadline, retry or first-frame rule. +A separate control discards only the Viewer's RTP/RTCP datagrams while allowing +STUN and DTLS. Both endpoints connect and the exact subscription binds, but no +Viewer frames arrive; the existing 20-second operation deadline then expires. +Thus even the reported timeout order does not identify a unique cause. Matched +Viewer transport/decode evidence is still required. Neither control changes a +product deadline, retry or first-frame rule. ## Browser Prediction Preflight @@ -582,6 +589,9 @@ candidate without another timer or readiness probe. Each retained flow is serial overlapping connections still use separate flows. Three local race controls keep sibling audio/video flowing through two retained-flow replacements and another flow's rollback; peer revocation stops fresh media on both remaining flows. +The combined listener-state, Tailcat/WireGuard revocation and bounded-TWCC +candidates also pass Linux race checks for those isolation boundaries and the +retained-flow/64-check nine-/eleven-second, permanent-loss and retirement cases. These checks do not define room-grant integration or establish full App interoperability or any additional direct-reachability benefit. diff --git a/docs/todo.md b/docs/todo.md index e437c7331..803a12f26 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -126,9 +126,10 @@ of a reporter's cause; research owns the completed experiments and their limits. `SFU_ONLY`. They still lack the matched Viewer ICE, decode and `route-ready` trace. The cited Host failure follows the operation deadline; healthy ingress and a bound subscription do not establish delivered frames or its cause. - Local Native H264 and [kernel-NAT Browser SFU controls](./research/nat-traversal.md#sfu-behind-server-nat) - pass, but do not reproduce the reporter's network. The blocked-Viewer control - preserves healthy ingress and a bound track without usable Viewer media. + Native H264 and Browser [SFU controls behind kernel NAT](./research/nat-traversal.md#sfu-behind-server-nat) + pass, but do not reproduce the reporter's network. Blocking Viewer media alone + preserves connected endpoints and a bound track until the operation deadline; + that log sequence does not identify a unique cause. Correlate transport-state diagnostics with the affected Viewer before changing admission or recovery. [#468](https://github.com/TNTcraftHIM/Piik/issues/468) needs the complete join From fa01a00d78c90b10f27120dfc1434c7b93eec922 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:00:28 +0800 Subject: [PATCH 060/262] docs(transport): record full-size media boundary --- docs/research/nat-traversal.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 1331fa0d2..9f3f50e46 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -410,11 +410,16 @@ Three race repetitions, the copied Tailcat listener suite and the combined revocation/isolation controls pass. The patch remains experimental; it neither changes Piik nor fixes the independent inner-ICE recovery failure below. -A separate real Chrome control decoded H.264/Opus and VP8/Opus from a generated -640×360 picture and tone through Browser WebRTC → Pion/Tailcat/Pion → Browser -WebRTC without re-encoding. Picture and decoded audio continued after DERP -closure and stopped after peer revocation. It uses ordinary Browser WebRTC, -not Browser-native Magicsock, physical capture or cross-browser acceptance. +Real Browser controls forward generated H.264/Opus and VP8/Opus through Browser +WebRTC → Pion/Tailcat/Pion → Browser WebRTC without re-encoding. Both decode at +640×360 and, with the combined dependency candidate, 1920×1080. The larger input +requests 60 fps with a 6 Mbps ceiling and fixed dimensions; actual encoding +cadence varies, so it establishes neither 1080p60 delivery nor a throughput +advantage. Fresh picture/audio arrive in each two-second window across eight +seconds after DERP closure and stop after revocation. At about 3.6–4.3 Mbps, +reported RTP loss is zero and the largest Tailcat datagram is 1208 bytes, with +no size rejection. This does not bound all RTCP workloads or establish physical +capture, Browser-native Magicsock or complete Piik relay interoperability. The 1200-byte Native packetization envelope with TWCC and short MID/RID headers produced at most 1228-byte encrypted datagrams across Pion's three default SRTP From d2195efced2383bed68fffdb869729b67bfc53c9 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:13:23 +0800 Subject: [PATCH 061/262] docs(network): verify published SFU startup behind NAT --- docs/research/nat-traversal.md | 7 +++++++ docs/todo.md | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 9f3f50e46..51ff14795 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -39,6 +39,13 @@ NAT, decoding 300 720p frames in Chrome 154, changing to 480p, recovering its subscription and reauthenticating after local media loss. The App captures a dedicated generated window; this does not reproduce a phone or carrier network. +The published v1.9.2 Windows App/capture and Linux Server artifacts, verified +against their release hashes and paired with that revision's Browser source, +also pass this Native H264/Opus startup control: 300 decoded 720p frames, a 480p +change, subscription recovery and retirement. This original-release control +excludes reauthentication after lost local media, which has a separate confirmed +repair below. It does not establish the cause of the reported phone failure. + Blocking only the Viewer process's UDP preserves connected Host ingress and LiveKit's `downtrack bound`, but the Viewer remains connecting without RTP or decoded frames. Its transport fails after about 15 seconds and the controller diff --git a/docs/todo.md b/docs/todo.md index 803a12f26..4d06e6cc7 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -127,7 +127,8 @@ of a reporter's cause; research owns the completed experiments and their limits. trace. The cited Host failure follows the operation deadline; healthy ingress and a bound subscription do not establish delivered frames or its cause. Native H264 and Browser [SFU controls behind kernel NAT](./research/nat-traversal.md#sfu-behind-server-nat) - pass, but do not reproduce the reporter's network. Blocking Viewer media alone + pass, including Native H264 on published v1.9.2, but do not reproduce the + reporter's network. Blocking Viewer media alone preserves connected endpoints and a bound track until the operation deadline; that log sequence does not identify a unique cause. Correlate transport-state diagnostics with the affected From 30e60276d33e649f96bb9391e2b15758d179c82a Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:19:12 +0800 Subject: [PATCH 062/262] docs(media): record audio preset compatibility gate --- docs/todo.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/todo.md b/docs/todo.md index 4d06e6cc7..a1861e71b 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -43,12 +43,17 @@ Existing negative results narrow the next experiment; they do not freeze the whole candidate or require another version-number decision before other work. Implementation needs a concrete useful result within the product model. -- [ ] **4K and additional codecs.** The [media assessment](./research/native-client-media.md#media-capability-extension-assessment) +- [ ] **4K, additional codecs and audio presets.** The [media assessment](./research/native-client-media.md#media-capability-extension-assessment) recommends retaining current limits. Assess concrete source/receiver cases before accepting coordinated capture, forwarding and resource-bound changes; keep UI presets separate from those bounds in any new design. Additional formats must account for late Viewers and relays that cannot decode or forward the published format; codec negotiation alone does not transcode it. + Review [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s 256/320 kbps Opus + presets under the same compatibility boundary: v1.9.2 rejects the new enums + in Viewer authentication and Native control, terminating those sessions. + Establish an audible benefit and a supported mixed-version policy before + adopting its additional controls; its current non-breaking claim is incorrect. - [ ] **Native Magicsock.** Go reuse is feasible, but the [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) have not established improved direct reachability. Seek a repeatable benefit From 5956830cc4004ec87178ab54da4f4f41e41d68cf Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:20:12 +0800 Subject: [PATCH 063/262] docs: retire resolved reports from pending work Retire the owner-confirmed deployment incident and stale closed-issue follow-ups. Consolidate device acceptance references while retaining independent reports and the limits of the SFU controls. --- docs/research/nat-traversal.md | 10 +++--- docs/todo.md | 61 ++++++++++------------------------ 2 files changed, 23 insertions(+), 48 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 51ff14795..7a0cba49f 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -44,7 +44,8 @@ against their release hashes and paired with that revision's Browser source, also pass this Native H264/Opus startup control: 300 decoded 720p frames, a 480p change, subscription recovery and retirement. This original-release control excludes reauthentication after lost local media, which has a separate confirmed -repair below. It does not establish the cause of the reported phone failure. +repair below. The desktop control does not establish physical phone or carrier +network coverage. Blocking only the Viewer process's UDP preserves connected Host ingress and LiveKit's `downtrack bound`, but the Viewer remains connecting without RTP or @@ -54,9 +55,9 @@ healthy ingress cannot establish Viewer readiness or a controller defect. A separate control discards only the Viewer's RTP/RTCP datagrams while allowing STUN and DTLS. Both endpoints connect and the exact subscription binds, but no Viewer frames arrive; the existing 20-second operation deadline then expires. -Thus even the reported timeout order does not identify a unique cause. Matched -Viewer transport/decode evidence is still required. Neither control changes a -product deadline, retry or first-frame rule. +Connected endpoints, a bound track and an operation timeout do not identify a +unique cause; diagnosis requires the affected Viewer's transport/decode evidence. +Neither control changes a product deadline, retry or first-frame rule. ## Browser Prediction Preflight @@ -550,7 +551,6 @@ product, while preserving the parent's recovery budget. Renewing that budget on replacement hid the nine-second outage: three repetitions recovered only after a third automatic request, outside the two-request contract. The corrected owner and fixture retain the count across Native and Browser backend replacement. -This independent recovery defect does not establish the self-hosted SFU report's cause. The actual-owner fixture now supports incremental SDP/candidate exchange, matching Piik's [Trickle ICE generation boundary](https://www.rfc-editor.org/rfc/rfc8838.html#section-13). diff --git a/docs/todo.md b/docs/todo.md index a1861e71b..4c3e7c10b 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -95,19 +95,15 @@ of a reporter's cause; research owns the completed experiments and their limits. boundary. Visible permission changes, display-mode changes and the reporter's setup still need matched physical evidence; API success does not prove that Windows hid the border. -- [ ] **macOS and Linux physical capture.** Complete the remaining capture/audio - checks in [verification status](./verification-status.md#device-evidence-boundary), - including an Intel Mac with a usable hardware H.264 encoder, an ARM64 Linux - desktop, and Linux systems beyond the PR author's Hyprland/NVIDIA result. -- [ ] **Signaling recovery on passwordless sites (#446).** Verify the admission - correction against a matched Host/Server attempt; identify the event preceding - the handshake failure rather than inferring password denial from `AUTH_REQUIRED`. -- [ ] **HDR, audio-exclusion and mobile interaction device coverage.** Complete - the [physical matrix](./verification-status.md#device-evidence-boundary): - HDR/mixed displays, voice-app process trees across playback devices and phone - keyboard/theater/participant-menu interaction. Browser HDR overexposure is an - accepted limitation with [documented alternatives](./guide/troubleshooting.md#hdr-picture-looks-too-bright-or-washed-out); - reopen conversion work only with new evidence and an accepted implementation. +- [ ] **Physical device and network acceptance.** Complete the remaining + [device checks](./verification-status.md#device-evidence-boundary) and + [network/endurance matrix](./verification-status.md#remaining-device-and-network-acceptance), + preserving platform deferrals and serial physical checks. Include macOS/Linux + capture, HDR/mixed displays, audio exclusion across real process trees and + playback devices, and mobile keyboard/theater/participant controls. The + [camera and microphone assessment](./research/camera-and-microphone.md) + additionally owns levels, echo, device replacement/native mixing, OBS input + A/V synchronization and phone camera permission/orientation/background limits. - [ ] **Native H.264 motion quality (#432).** Obtain matched source/decoded pictures and bitstream for the original 720p30 game comparison. Retain current codec parameters. The [evidence review](./research/native-client-media.md#native-h264-motion-quality) @@ -126,18 +122,6 @@ of a reporter's cause; research owns the completed experiments and their limits. [#453](https://github.com/TNTcraftHIM/Piik/issues/453) has no matched endpoint reports to distinguish network reachability from a product failure. The v1.6.3 candidate-timeout report lacks share-start and SDP/ICE evidence. - [#464](https://github.com/TNTcraftHIM/Piik/issues/464) and the supplied report - identify v1.9.2, a NAT-hosted Server, Native Host, and the same timeout under - `SFU_ONLY`. They still lack the matched Viewer ICE, decode and `route-ready` - trace. The cited Host failure follows the operation deadline; healthy ingress - and a bound subscription do not establish delivered frames or its cause. - Native H264 and Browser [SFU controls behind kernel NAT](./research/nat-traversal.md#sfu-behind-server-nat) - pass, including Native H264 on published v1.9.2, but do not reproduce the - reporter's network. Blocking Viewer media alone - preserves connected endpoints and a bound track until the operation deadline; - that log sequence does not identify a unique cause. - Correlate transport-state diagnostics with the affected - Viewer before changing admission or recovery. [#468](https://github.com/TNTcraftHIM/Piik/issues/468) needs the complete join trace to distinguish a new Viewer from reauthentication with lost local media; its excerpt starts with an already committed SFU edge. Retest the affected @@ -154,11 +138,6 @@ of a reporter's cause; research owns the completed experiments and their limits. reports before changing routing or advising users to disable IP masking. [#463](https://github.com/TNTcraftHIM/Piik/issues/463) lacks the Browser-only Mac/phone attempt details needed to distinguish capture, entry and media failure. -- [ ] **Camera and Host microphone device coverage.** Check real audio levels, - echo, device replacement/native mixing and phone camera permission, orientation - and background behavior. Include OBS virtual-camera/audio-input sound quality - and A/V synchronization. The [capture assessment](./research/camera-and-microphone.md) - owns current behavior and the accepted physical limits. - [ ] **Public-invitation startup and HTTP 1033.** Obtain version and matched diagnostics for creation failures, Demo access and a stream stalling before reload returns 1033. Distinguish connector recovery/exit, DNS/provider failure @@ -174,8 +153,7 @@ of a reporter's cause; research owns the completed experiments and their limits. screenshot. Retaining dependency failure details improves the next report; it does not establish or resolve this attempt's cause. - [ ] **Viewer interruptions.** Obtain paired endpoint/upstream-relay diagnostics - for established viewing returning to P2P connecting, including - [#429](https://github.com/TNTcraftHIM/Piik/issues/429)'s SFU-to-P2P dropout. + for established viewing returning to P2P connecting. Separately retest the v1.5.0 App public-link report where one Viewer drops immediately after connection details appear while others work. Distinguish first-frame admission, signaling grace and current-edge recovery. @@ -228,35 +206,32 @@ works, so a Rust alternative needs a concrete Go limitation. ## Parked Product Work -1. **Representative device/network acceptance.** Resume the remaining matrix in - [verification status](./verification-status.md#remaining-device-and-network-acceptance) - when directed, preserving platform deferrals and serial physical checks. -2. **Broader quality work.** Reopen from measured interruptions, failed recovery +1. **Broader quality work.** Reopen from measured interruptions, failed recovery or healthy-sibling degradation at acceptable complexity. The [Browser pool comparison](./research/browser-local-encoding-pool.md#balanced-startup-and-recovery) owns cold-encoder tradeoffs; low resolution alone is not a defect. Check whether a proposed move merely shifts pressure to another parent's siblings. -3. **Platform output.** Reopen for a registered receiver acting as an ordinary +2. **Platform output.** Reopen for a registered receiver acting as an ordinary Viewer only after the [platform-output gate](./research/platform-output.md) passes. -4. **Additional languages.** Review community catalogs and their rendered UI +3. **Additional languages.** Review community catalogs and their rendered UI following the [translation guide](./guide/translating.md), including names, menu navigation, text direction and layout. -5. **Windows code signing.** Revisit after enrollment in a trusted signing +4. **Windows code signing.** Revisit after enrollment in a trusted signing service. Sign executables before archive checksums; publisher identity does not guarantee that antivirus cloud scanning stops. -6. **Gitee download-source warning.** Paused by the owner. Keep GitHub primary +5. **Gitee download-source warning.** Paused by the owner. Keep GitHub primary and retain Gitee; do not add a self-hosted mirror. Chrome still blocks the Gitee attachment when Referer is removed. Reopen for new evidence or a provider review; the warning remains unresolved. -7. **Release-operation policy.** Protected release environments and immutable +6. **Release-operation policy.** Protected release environments and immutable draft assets remain unaccepted proposals; evaluate their benefit before adding release machinery. -8. **Automatic local chat history.** Explicit TXT export covers manual retention. +7. **Automatic local chat history.** Explicit TXT export covers manual retention. Reconsider automatic storage only with a stable room-incarnation identity, bounded retention and a clear delete control; reusable room codes must not combine conversations. No new history service or wire field is authorized. -9. **Direct OBS input (#436).** Reopen for a measured virtual-camera limitation. +8. **Direct OBS input (#436).** Reopen for a measured virtual-camera limitation. The [capture assessment](./research/camera-and-microphone.md) records raw-frame output and the WHIP boundary; virtual camera alone is not a second encode. Do not add an RTMP/WHIP listener without an accepted publication owner. From 650953f17a1ee6d591a8f3c65046de533495c401 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:35:03 +0800 Subject: [PATCH 064/262] docs(media): plan bounded capability design for 2.0 Hold maintenance delivery for the 2.0 candidate. Separate UI presets, numeric requests, capabilities and actual media in the proposed design; assess audio PR 462 through that boundary and retain the incomplete Magicsock integration and external-test gates. --- docs/research/native-client-media.md | 99 ++++++++++++++++++++++------ docs/status.md | 13 ++-- docs/todo.md | 39 +++++++---- 3 files changed, 111 insertions(+), 40 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 003f617ad..85dc24b8a 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -510,8 +510,10 @@ a corrected SDP interpretation, not evidence that all Firefox sharing lost audio ## Media Capability Extension Assessment -Proposal assessed 2026-10-04 for a possible 2.0 phase; this does not change the -current contract or the profile decision above. Resolution enums are enforced +Planning direction reviewed 2026-10-09 for 2.0; the design below is a proposal, +not a replacement for the current contract or profile decision above. The owner +reopened numeric media requests and capability negotiation as one coherent +extension. Resolution enums are enforced by both signaling readers and Native control, while capture admission separately bounds dimensions. Native capture, forwarding and SFU declarations also admit specific encoded formats. Removing one validator would leave these consumers @@ -535,21 +537,45 @@ These are different meanings. Keep explicit resource ceilings, but assess them separately from presets and device capability; arbitrary numbers are not the replacement contract. -The smallest proposed extension separates UI presets from numeric media limits -and uses existing WebRTC codec negotiation. The -[Media Capture specification](https://www.w3.org/TR/mediacapture-streams/) -distinguishes requested constraints, supported ranges and actual settings; -[WebRTC capabilities](https://www.w3.org/TR/webrtc/#rtcrtpsender-interface) -advertise optimistic codec support, not guaranteed hardware execution. Preserve -actual encoder admission and validate dimensions, pixel counts, frame sizes and -resource bounds at their existing owners. +The proposed boundary follows the +[Media Capture specification](https://www.w3.org/TR/mediacapture-streams/#constrainable-interface): +requested constraints, supported capabilities and actual settings have different +owners. Capability ranges cannot describe every combination of resolution and +frame rate; [WebRTC codec discovery](https://www.w3.org/TR/webrtc/#rtcrtpsender-interface) +is optimistic and reserves no hardware resources. An advertised capability is +therefore not proof that a particular encoder configuration can start. + +| Meaning | Proposed Piik owner and representation | +| --- | --- | +| Quick presets | UI-local labels mapping to numeric picture/audio requests; adding a label does not extend the wire | +| Host intent | One validated numeric size, frame-rate and bitrate request, distinct from measured output | +| Available capability | Existing capture/receive/forward adapters report supported operations and bounded formats; unknown descriptive formats are unavailable | +| Applied configuration | The existing share owner commits an admitted result under its current generation; failed or stale changes do not become saved preferences | +| Live output | Existing capture and receiver evidence; WebRTC/LiveKit still own network adaptation | + +Dimensions, pixel counts/rates, encoded-frame bytes, allocations and concurrent +outputs remain bounded before resource use. Remote capability claims cannot raise +local limits. Reject malformed values and unauthorized commands; unsupported +well-formed requests need a scoped failure, not an unrelated room disconnect. +Use the existing generation and route operation, not a capability polling loop, +general constraint solver or second recovery owner. Numeric limits describe +ceilings, not a promise of a constant rate or resolution. A receiver's limits +constrain its edge or existing local output group, not every Viewer in the room. Codec support must distinguish encoding, decoding and encoded forwarding, -including profile/packetization and SFU admission. AV1 and VP9 are candidates to +including profile/packetization and SFU admission. Reuse SDP codec matching; +describe only Native control and forwarding limits that it cannot express, +instead of negotiating the same format twice. AV1 and VP9 are candidates to evaluate, not promised product formats. A late Viewer or relay that cannot receive the current encoded source needs an explicit policy; SDP negotiation alone cannot convert that source. Per-Viewer transcodes or room-wide codec -switches are not implied by this proposal. +switches are not implied by this proposal. This is also explicit in +[mediasoup's producer/consumer capabilities](https://mediasoup.org/documentation/v3/mediasoup/rtp-parameters-and-capabilities/). +[LiveKit's backup codec policy](https://github.com/livekit/client-sdk-js/blob/main/src/room/track/options.ts) +uses another encoding or switches the publication; that compatibility has real +CPU/bandwidth or continuity costs. Piik's embedded LiveKit components do not +automatically provide its complete SDK policy. Keep H.264/VP8 as the initial +compatibility path; admit a new format only with its whole-route evidence. Current discovery couples format to implementation (`hardwareH264` and `softwareVP8`), and Browser preference uses a VP8-specific fallback flag. @@ -568,15 +594,46 @@ inherently require a major; breaking an existing public contract does. Evaluate the Browser/App/Server and mixed-peer flows before freezing a replacement contract; avoid a general compatibility framework. -The 2026-10-07 candidate review does not recommend adding these extensions to -the current release. A 4K path must change the three platform capture/decoder -bounds, encoded-frame limits and SFU admission together; Windows H.264's current -level ceiling also does not cover 4K60. The existing Main/High comparison shows -a modest synthetic quality gain, but Firefox's receiver gap prevents treating -it as a transparent upgrade. Keep the current formats and limits until a -specific use case justifies that coordinated change. This is a scope -recommendation, not a claim that 4K is impossible or that the current geometry -should become a permanent protocol rule. +### Audio And UI Proposal + +[#462](https://github.com/TNTcraftHIM/Piik/pull/462) requests 256/320 kbps Opus. +The values are permitted by [RFC 7587](https://www.rfc-editor.org/rfc/rfc7587.html#section-6.1), +whose receive-rate parameter spans 6,000..510,000 bit/s. This is a codec boundary, +not a recommended default or proof of audible benefit. The current preset names +also travel in authentication and Native commands: adding its two enum values +causes the actual v1.9.2 readers to terminate those sessions. The PR's compatible +release claim is therefore incorrect even though its audio use case is valid. + +For 2.0, prototype one numeric audio-bitrate ceiling through capture, sender, +receiver and SFU; derive existing preset labels locally. Align receive admission +with codec support instead of the largest UI shortcut. Keep the existing default +while comparing actual Opus output and listening results for higher ceilings. +[LiveKit's AudioPreset](https://docs.livekit.io/reference/client-sdk-js/interfaces/AudioPreset.html) +likewise contains a numeric maximum bitrate; its +[high-fidelity example](https://docs.livekit.io/transport/media/advanced/#hi-fi-audio) +also considers stereo, source processing and redundancy. Raising bitrate alone +does not establish studio quality, losslessness or multi-channel output. + +Keep Piik's compact presets and existing picture/sound settings disclosure. +Expose custom numeric values with units and valid ranges there, without adding +a permanent row for each new preset. Use supported-format selection for codecs, +not an arbitrary text field. Preserve source aspect ratio without upscaling; +distinguish selected limits from actual capture/decoded dimensions in existing +details. Capability-related restrictions explain a real limitation briefly. +The [OBS settings guide](https://obsproject.com/kb/obs-studio-overview#settings) +supports a simple/default entry with further controls; Piik need not reproduce +OBS's full output configuration. Existing visual/tooltip owners still apply, +including narrow screens, both themes and visual mode. + +Start with numeric audio through the existing pipeline, then portrait/ultrawide +geometry and 4K as a second concrete use case. A 4K path must review all platform +capture/decoder bounds, encoded-frame limits and SFU admission; Windows H.264's +current level ceiling does not cover 4K60. Admission and output must be tested +at the actual profile, not inferred from a codec list. Check live changes, +late/limited Viewers and relays, failure cleanup and resource rejection before +freezing the 2.0 contract. Prefer a clear 1.x mismatch over a general legacy +adapter. Future presets inside supported numeric semantics need no new enum; +new formats still require actual implementation and an advertised capability. ## Implementation Boundary diff --git a/docs/status.md b/docs/status.md index 60690fee5..5e3507f31 100644 --- a/docs/status.md +++ b/docs/status.md @@ -1,6 +1,6 @@ # Current Status -Last updated: 2026-10-07 +Last updated: 2026-10-09 This is the compact execution/deployment index. Product modules own behavior, [verification status](./verification-status.md) owns unresolved physical limits, @@ -59,10 +59,13 @@ SQLite schema 2 and the 1440p ceiling. Hosted empty-room retention and capacity guidance preserve indefinite retention by default. The accepted Demo policy is one hour without authenticated participants; the private deployment retains the default. Operator records own activation, database and postflight evidence. -The owner reopened 2.0 candidate preparation, including reassessment of 4K, -codec expansion and Native Magicsock where demonstrated benefit justifies total -complexity. These experiments do not yet change the accepted contract or product -transport. Release remains subject to the owner's final acceptance. +The owner is holding the unpublished maintenance changes for 2.0. Preparation +includes separating media presets from numeric requests and bounded capabilities, +with audio presets, 4K and additional codecs assessed through that boundary. +Native Magicsock remains an isolated experiment with incomplete product +integration and unresolved recovery cases. These proposals do not yet change +the accepted contract or transport. A small external test is proposed before +release; publication remains subject to the owner's final acceptance. Immutable release artifacts own delivery completion; [TODO](./todo.md#now) owns remaining work. diff --git a/docs/todo.md b/docs/todo.md index 4c3e7c10b..e699d9003 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -11,9 +11,10 @@ Git/PRs own completed history. ## Now - [ ] **2.0 candidate preparation and acceptance.** The owner reopened investigation, - useful improvements and outstanding acceptance on 2026-10-07. Prepare one - coherent candidate and stop before publication. Preserve the existing product - model; evaluate held features against measurable benefit and total complexity + useful improvements and outstanding acceptance on 2026-10-07, and on 2026-10-09 + chose to hold the maintenance changes for 2.0 rather than publish them separately. + Prepare one coherent candidate and stop before publication. Preserve the existing + product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance are complete. Integrate through the next owner-approved publishing merge and @@ -43,17 +44,18 @@ Existing negative results narrow the next experiment; they do not freeze the whole candidate or require another version-number decision before other work. Implementation needs a concrete useful result within the product model. -- [ ] **4K, additional codecs and audio presets.** The [media assessment](./research/native-client-media.md#media-capability-extension-assessment) - recommends retaining current limits. Assess concrete source/receiver cases - before accepting coordinated capture, forwarding and resource-bound changes; - keep UI presets separate from those bounds in any new design. Additional - formats must account for late Viewers and relays that cannot decode or forward - the published format; codec negotiation alone does not transcode it. - Review [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s 256/320 kbps Opus - presets under the same compatibility boundary: v1.9.2 rejects the new enums - in Viewer authentication and Native control, terminating those sessions. - Establish an audible benefit and a supported mixed-version policy before - adopting its additional controls; its current non-breaking claim is incorrect. +- [ ] **Media parameters and capabilities for 2.0.** Follow the + [media assessment](./research/native-client-media.md#media-capability-extension-assessment): + separate UI presets, numeric requests, bounded implementation capabilities and + actual output. Prototype numeric audio bitrate first using + [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s use case, then + source-aspect-preserving geometry and 4K through the same owners. Preserve + safety limits and framework-owned adaptation. New codecs need complete + encode/decode/forward support and a late-Viewer policy before inclusion; + selecting a codec cannot transcode an existing publication. Confirm live + changes, relay/SFU paths, older 2.x capability subsets and explicit 1.x + mismatch before accepting the replacement contract. The current 1.x enums + remain; the PR is not a compatible enum-only extension of v1.9.2. - [ ] **Native Magicsock.** Go reuse is feasible, but the [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) have not established improved direct reachability. Seek a repeatable benefit @@ -68,6 +70,15 @@ Implementation needs a concrete useful result within the product model. media-readiness authority. Research owns the experiments and their limits. No timeout change or transport integration is accepted; the P2P evidence work below owns the network/interop acceptance boundary. +- [ ] **Small external test before 2.0.** First use the isolated transport + experiment on the same difficult network pairs as current Piik, measuring + direct success separately from relay delivery, first-picture time, recovery + and added resource cost. This evidence can precede other feature work. + A combined product test package requires room-scoped authorization/revocation, + mixed Browser/Native behavior and bounded recovery to pass first. Freeze and + identify each test build; do not describe the present Magicsock experiment as + a feature-complete product awaiting only acceptance. The final candidate then + receives the combined feature/UI and regression check. - [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) identifies UDP/storage requirements but establishes no cost saving. Keep the existing P2P-only Demo until a provider and operating boundary are accepted. From 3b47b50c65956e6614d82271939d816253a4b9ad Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:12:11 +0800 Subject: [PATCH 065/262] refactor(media): decouple audio sender limits from presets --- docs/research/browser-screen-audio-quality.md | 39 +++++++++++++++++++ docs/research/native-client-media.md | 3 ++ docs/todo.md | 8 +++- internal/app/nativeaudio/encoder_test.go | 30 ++++++++++++-- src/client/media/quality.ts | 3 +- src/client/sfu/publisher.ts | 2 +- src/client/webrtc/host-peer.ts | 3 +- tests/quality.test.ts | 18 ++++----- 8 files changed, 87 insertions(+), 19 deletions(-) diff --git a/docs/research/browser-screen-audio-quality.md b/docs/research/browser-screen-audio-quality.md index 1aaad6a70..a856332aa 100644 --- a/docs/research/browser-screen-audio-quality.md +++ b/docs/research/browser-screen-audio-quality.md @@ -55,6 +55,45 @@ weak-network behavior, real-game A/V synchronization, or mobile playback. - Lossy Wi-Fi/cellular concealment and Opus FEC behavior with current options. - macOS, Android, and iOS capture/playback behavior. +## Numeric Bitrate Assessment + +Checked 2026-10-09 for the proposed 2.0 media boundary. The current product still +uses its published presets; these experiments do not accept a new wire format or +raise its audio ceiling. The shared Browser sender primitive now takes a numeric +ceiling, with current direct/SFU callers resolving their existing preset first. + +The existing Go Opus encoder accepted live 128/192/256/320/96/128 kbps changes +without replacement. A 48 kHz stereo tone/noise input continued decoding into +960 samples per channel; the existing deterministic tests also verify stereo +separation at 256/320 kbps and rejection of invalid updates without replacing +the last setting. This establishes encoding continuity, not audible benefit. + +A Chrome 152 loopback used the actual sender primitive, generated stereo audio, +and decoded PCM measurement. The same PeerConnections remained connected while +changing those ceilings. Received payload tracked approximately 96/128/192/256/320 +kbps, with decoded samples advancing and nonzero waveform energy. This is a +local Browser check, not relay/SFU, mobile, impaired-network or listening acceptance. + +Two controls constrain the replacement design: + +- Chrome retained `maxaveragebitrate=192000` in the remote answer and sender + codec parameters, yet later `setParameters()` calls produced approximately + 256/320 kbps. Repeating with a 510000 answer also passed. This scoped runtime + counterexample does not change [RFC 7587's receive-limit semantics](https://www.rfc-editor.org/rfc/rfc7587.html#section-7.1). + A numeric sender setting must respect the admitted receiver limit; do not + assume SDP presence or successful parameter readback proves enforcement. +- With native 20 ms frames, the tone/noise check produced maximum Opus payloads + of 641 bytes at 256 kbps and 801 at 320 kbps. At 510 kbps, all 150 frames + produced RTP packets larger than a 1200-byte packetizer budget. Pinned + [Pion's Opus payloader](https://github.com/pion/rtp/blob/v1.10.2/codecs/opus_packet.go) + keeps each Opus payload intact. This is a packet-size result, not measured + network loss; accepting the codec's maximum requires a transport/packet-duration + decision. Do not equate the codec range with every route's safe operating range. + +The [media extension proposal](./native-client-media.md#audio-and-ui-proposal) +owns the planned UI and contract. Complete admitted receive limits, native control, +preference storage and live relay/SFU verification before exposing custom values. + ## Primary Sources - [Screen Capture](https://www.w3.org/TR/screen-capture/) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 85dc24b8a..7beb9cde6 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -608,6 +608,9 @@ For 2.0, prototype one numeric audio-bitrate ceiling through capture, sender, receiver and SFU; derive existing preset labels locally. Align receive admission with codec support instead of the largest UI shortcut. Keep the existing default while comparing actual Opus output and listening results for higher ceilings. +The [numeric audio controls](./browser-screen-audio-quality.md#numeric-bitrate-assessment) +establish local 256/320 kbps encoding and Browser reception, with separate +receive-limit enforcement and high-rate packet-size boundaries still to resolve. [LiveKit's AudioPreset](https://docs.livekit.io/reference/client-sdk-js/interfaces/AudioPreset.html) likewise contains a numeric maximum bitrate; its [high-fidelity example](https://docs.livekit.io/transport/media/advanced/#hi-fi-audio) diff --git a/docs/todo.md b/docs/todo.md index e699d9003..b12a638dd 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -47,10 +47,14 @@ Implementation needs a concrete useful result within the product model. - [ ] **Media parameters and capabilities for 2.0.** Follow the [media assessment](./research/native-client-media.md#media-capability-extension-assessment): separate UI presets, numeric requests, bounded implementation capabilities and - actual output. Prototype numeric audio bitrate first using + actual output. Complete numeric audio bitrate admission and storage first using [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s use case, then source-aspect-preserving geometry and 4K through the same owners. Preserve - safety limits and framework-owned adaptation. New codecs need complete + safety limits and framework-owned adaptation. The + [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment) + require explicit receive-limit enforcement and a packet-size decision before + exposing higher numeric ceilings; add live relay/SFU acceptance to the local + encoding/Browser evidence. New codecs need complete encode/decode/forward support and a late-Viewer policy before inclusion; selecting a codec cannot transcode an existing publication. Confirm live changes, relay/SFU paths, older 2.x capability subsets and explicit 1.x diff --git a/internal/app/nativeaudio/encoder_test.go b/internal/app/nativeaudio/encoder_test.go index 074405420..fb8008aad 100644 --- a/internal/app/nativeaudio/encoder_test.go +++ b/internal/app/nativeaudio/encoder_test.go @@ -86,7 +86,7 @@ func TestEncodePreservesAudiblePCM(t *testing.T) { } func TestEncodePreservesStereoSeparation(t *testing.T) { - for _, bitrate := range []int{64_000, 128_000, 192_000} { + for _, bitrate := range []int{64_000, 128_000, 192_000, 256_000, 320_000} { encoder, err := NewEncoder(bitrate) if err != nil { t.Fatal(err) @@ -150,13 +150,35 @@ func TestEncoderUpdatesBitrateWithinTheSameGeneration(t *testing.T) { if err != nil { t.Fatal(err) } - for _, bitrate := range []int{64_000, 192_000, DefaultBitrate} { + decoder, err := gopus.NewDecoder(gopus.DefaultDecoderConfig(SampleRate, Channels)) + if err != nil { + t.Fatal(err) + } + pcm := make([]byte, FrameBytes) + decoded := make([]int16, FrameSamples*Channels) + for _, bitrate := range []int{64_000, 192_000, 256_000, 320_000, 96_000, DefaultBitrate} { if err = encoder.SetBitrate(bitrate); err != nil { t.Fatalf("SetBitrate(%d) = %v", bitrate, err) } + if encoder.Bitrate() != bitrate { + t.Fatalf("SetBitrate(%d) readback = %d", bitrate, encoder.Bitrate()) + } + packet, err := encoder.Encode(pcm) + if err != nil { + t.Fatal(err) + } + count, err := decoder.DecodeInt16(packet, decoded) + if err != nil || count != FrameSamples { + t.Fatalf("decode after bitrate %d: %d samples, %v", bitrate, count, err) + } } - if err = encoder.SetBitrate(0); err == nil { - t.Fatal("invalid live bitrate was accepted") + for _, bitrate := range []int{0, 5_999, 510_001} { + if err = encoder.SetBitrate(bitrate); err == nil { + t.Fatalf("invalid live bitrate %d was accepted", bitrate) + } + if encoder.Bitrate() != DefaultBitrate { + t.Fatalf("rejected bitrate %d replaced the active setting", bitrate) + } } } diff --git a/src/client/media/quality.ts b/src/client/media/quality.ts index cc5602a5c..bf98bf565 100644 --- a/src/client/media/quality.ts +++ b/src/client/media/quality.ts @@ -400,7 +400,7 @@ export async function configureVideoSender( export async function configureScreenAudioSender( sender: RTCRtpSender, - quality?: ScreenAudioQuality, + requestedMaxBitrate: number, ): Promise { const current = sender.getParameters(); const parameters = { @@ -410,7 +410,6 @@ export async function configureScreenAudioSender( if (parameters.encodings.length === 0) { parameters.encodings = [{}]; } - const requestedMaxBitrate = screenAudioBitrate(quality); parameters.encodings[0]!.maxBitrate = requestedMaxBitrate; await sender.setParameters(parameters); const appliedMaxBitrate = diff --git a/src/client/sfu/publisher.ts b/src/client/sfu/publisher.ts index bdc66d1fa..8f79aa124 100644 --- a/src/client/sfu/publisher.ts +++ b/src/client/sfu/publisher.ts @@ -510,7 +510,7 @@ export class SfuPublisher { try { const audio = await peer.waitForOperation(() => configureScreenAudioSender( audioSender, - profile.screenAudioQuality, + screenAudioBitrate(profile.screenAudioQuality), )); if (this.peer !== peer) return false; this.audioWarning = audioSenderParameterWarning(audio); diff --git a/src/client/webrtc/host-peer.ts b/src/client/webrtc/host-peer.ts index d89c399bf..ad8fbecbb 100644 --- a/src/client/webrtc/host-peer.ts +++ b/src/client/webrtc/host-peer.ts @@ -12,6 +12,7 @@ import { configureVideoSender, needsStartupVideoProfile, resolveScreenAudioQuality, + screenAudioBitrate, screenAudioQualityEqual, STARTUP_VIDEO_ENCODED_FRAMES, startupVideoProfile, @@ -924,7 +925,7 @@ export class HostPeer { try { audioSenderParameters = await this.waitForOperation(() => configureScreenAudioSender( audioSender, - profile.screenAudioQuality, + screenAudioBitrate(profile.screenAudioQuality), )); } catch (error) { if (this.disposed) return false; diff --git a/tests/quality.test.ts b/tests/quality.test.ts index c3c491789..12000d778 100644 --- a/tests/quality.test.ts +++ b/tests/quality.test.ts @@ -12,7 +12,7 @@ import { QUALITY_PROFILES, QUALITY_RESOLUTIONS, qualitySettingsEqual, - SCREEN_AUDIO_BITRATES, + screenAudioBitrate, senderParameterWarning, setMediaPaused, startupVideoProfile, @@ -294,6 +294,7 @@ describe("realtime quality controls", () => { }); it.each([ + [undefined, 128_000], ["saver", 64_000], ["music", 128_000], ["very-high", 192_000], @@ -306,16 +307,15 @@ describe("realtime quality controls", () => { }), } as unknown as RTCRtpSender; - await expect(configureScreenAudioSender(sender, quality)).resolves.toEqual({ + await expect(configureScreenAudioSender(sender, screenAudioBitrate(quality))).resolves.toEqual({ requestedMaxBitrate: bitrate, appliedMaxBitrate: bitrate, mismatch: false, }); expect(applied.encodings).toEqual([{ maxBitrate: bitrate }]); - expect(SCREEN_AUDIO_BITRATES[quality]).toBe(bitrate); }); - it("uses the music ceiling for a setting without an audio preset", async () => { + it.each([96_000, 256_000, 320_000])("applies a numeric %i audio ceiling without a preset", async (bitrate) => { let applied = { encodings: [] } as unknown as RTCRtpSendParameters; const sender = { getParameters: () => applied, @@ -324,9 +324,9 @@ describe("realtime quality controls", () => { }), } as unknown as RTCRtpSender; - await expect(configureScreenAudioSender(sender)).resolves.toEqual({ - requestedMaxBitrate: 128_000, - appliedMaxBitrate: 128_000, + await expect(configureScreenAudioSender(sender, bitrate)).resolves.toEqual({ + requestedMaxBitrate: bitrate, + appliedMaxBitrate: bitrate, mismatch: false, }); }); @@ -342,7 +342,7 @@ describe("realtime quality controls", () => { setParameters: vi.fn(async () => undefined), } as unknown as RTCRtpSender; - const readback = await configureScreenAudioSender(sender, "music"); + const readback = await configureScreenAudioSender(sender, 128_000); expect(readback).toEqual({ requestedMaxBitrate: 128_000, @@ -367,7 +367,7 @@ describe("realtime quality controls", () => { } as unknown as RTCRtpSender; await expect( - configureScreenAudioSender(sender, "very-high"), + configureScreenAudioSender(sender, 192_000), ).rejects.toThrow("rejected"); expect(applied.encodings[0]?.maxBitrate).toBe(128_000); }); From 38ed8b47e4765944fd2f3d2957840a1f88cde672 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:15:27 +0800 Subject: [PATCH 066/262] feat(media)!: support numeric audio bitrate ceilings Keep presets local to the UI and validate bounded numeric requests in the 2.0 candidate. Advertise and enforce Opus receive limits without reducing shared sources; preserve media continuity during audio-only updates. --- docs/project-memory.md | 6 +- docs/research/browser-screen-audio-quality.md | 33 ++++- docs/research/media-fidelity.md | 4 +- docs/research/native-client-media.md | 5 +- docs/standards/media-quality.md | 20 ++- docs/status.md | 10 +- docs/todo.md | 16 +- internal/app/app.go | 2 + .../app/config/quality_preference_test.go | 2 +- internal/app/loopback/protocol.go | 4 +- internal/app/loopback/protocol_test.go | 18 +-- internal/app/loopback/server.go | 1 + internal/app/mediaedge/edge.go | 6 + internal/app/mediaedge/edge_test.go | 6 +- internal/app/mediaedge/publication.go | 1 + internal/app/mediaedge/receiver.go | 3 +- internal/app/mediaedge/receiver_test.go | 2 +- internal/app/nativeaudio/encoder.go | 20 ++- .../nativecontrol/operation_failure_test.go | 28 ++-- internal/app/nativecontrol/session.go | 31 ++-- internal/app/nativecontrol/session_test.go | 62 ++++---- internal/app/nativecontrol/wire.go | 9 +- internal/app/nativehost/profile.go | 9 +- internal/media/forwarding/audio.go | 52 +++++++ internal/media/forwarding/audio_test.go | 34 +++++ internal/server/protocol/constants.go | 10 +- internal/server/protocol/contract_test.go | 8 +- internal/server/protocol/types.go | 9 +- internal/server/sfu/media.go | 9 +- internal/server/sfu/media_test.go | 12 ++ internal/server/signal/server_test.go | 12 +- scripts/app-media-gate.ts | 8 +- scripts/browser-local-pool-page.js | 5 +- scripts/browser-local-pool-probe.ts | 6 +- scripts/embedded-sfu-gate.ts | 14 ++ scripts/embedded-sfu-page.ts | 12 +- src/client/locales/en.ts | 4 +- src/client/locales/zh.ts | 4 +- src/client/media/quality.ts | 67 ++++++--- src/client/native/wire.ts | 7 +- src/client/pages/HostPage.tsx | 68 +++++++-- src/client/sfu/publisher.ts | 17 ++- src/client/styles.css | 2 + src/client/webrtc/host-peer.ts | 35 +++-- src/shared/protocol.ts | 15 +- tests/browser-debug.test.ts | 10 +- tests/client-state.test.ts | 4 +- tests/fixtures/wire-samples.json | 140 +++++++++++++----- tests/host-peer-tracks.test.ts | 36 ++++- tests/host-quality-ownership.test.ts | 4 +- tests/native-client.test.ts | 46 +++--- tests/native-media-bridge.test.ts | 14 +- tests/native-sender-edge.test.ts | 16 +- tests/native-sender-peer.test.ts | 4 +- tests/native-sfu-publisher.test.ts | 12 +- tests/native-viewer-peer.test.ts | 8 +- tests/protocol.test.ts | 22 +-- tests/quality-preference.test.ts | 2 +- tests/quality.test.ts | 43 +++++- tests/screen-audio-sdp.test.ts | 8 +- tests/sfu-client.test.ts | 4 +- tests/viewer-peer-generation.test.ts | 2 +- 62 files changed, 742 insertions(+), 341 deletions(-) create mode 100644 internal/media/forwarding/audio.go create mode 100644 internal/media/forwarding/audio_test.go diff --git a/docs/project-memory.md b/docs/project-memory.md index d49572375..4ca8d2b6a 100644 --- a/docs/project-memory.md +++ b/docs/project-memory.md @@ -1,6 +1,6 @@ # Project Memory -Last updated: 2026-10-06 +Last updated: 2026-10-09 Piik is private, low-latency screen sharing for one Host and up to 20 authenticated friends. The current product surface is Web Host, Web Viewer, and @@ -92,7 +92,9 @@ implementation and routine UI detail. periodic rebalance is accepted. - Browser video uses `motion`; each share chooses H.264 with VP8 fallback through an actual sender probe unless the Host explicitly selects VP8 or H264. Screen - audio uses 64/128/192 kbps ceilings with 128 default. + audio retains its compact presets and 128 kbps default. The 2.0 candidate + separates numeric bitrate requests from those UI shortcuts; implementation + limits and admission remain in the media-quality owner. - Viewer playback uses one native video element. Presentation and recovery are fenced by current route/media identity and current-frame proof; hidden-page time alone is not media-failure evidence or a keepalive guarantee. diff --git a/docs/research/browser-screen-audio-quality.md b/docs/research/browser-screen-audio-quality.md index a856332aa..02341fbcc 100644 --- a/docs/research/browser-screen-audio-quality.md +++ b/docs/research/browser-screen-audio-quality.md @@ -57,10 +57,10 @@ weak-network behavior, real-game A/V synchronization, or mobile playback. ## Numeric Bitrate Assessment -Checked 2026-10-09 for the proposed 2.0 media boundary. The current product still -uses its published presets; these experiments do not accept a new wire format or -raise its audio ceiling. The shared Browser sender primitive now takes a numeric -ceiling, with current direct/SFU callers resolving their existing preset first. +Checked 2026-10-09 for the 2.0 candidate. The published 1.x product retains its +preset contract. Candidate numeric requests and supported limits are owned by +[media quality](../standards/media-quality.md#screen-audio); these measurements +do not establish audible improvement or accept a public release. The existing Go Opus encoder accepted live 128/192/256/320/96/128 kbps changes without replacement. A 48 kHz stereo tone/noise input continued decoding into @@ -91,8 +91,29 @@ Two controls constrain the replacement design: decision. Do not equate the codec range with every route's safe operating range. The [media extension proposal](./native-client-media.md#audio-and-ui-proposal) -owns the planned UI and contract. Complete admitted receive limits, native control, -preference storage and live relay/SFU verification before exposing custom values. +owns the UI and contract rationale. The candidate enforces a Browser sender's +negotiated Opus limit, and admits unchanged Native/SFU forwarding only when the +receiver's declared limit accommodates the full shared format. A codec-valid +native setting beyond the current sender ceiling remains an operational failure. +Browser and App preference stores round-trip custom numeric values; those stores +were not part of published v1.9.2, so no legacy preset migration is required. + +The retained SFU gate passed for both Browser and Windows native publication: +live 320/96 kbps changes retained the same subscription, with advancing decoded +audio energy and video frames. Video profile changes, reload, recovery and +retirement also passed; the Browser run measured stereo separation. An ordinary +Browser relay forwarded received tracks to two children at 320 kbps, with +advancing decoded audio and successful cleanup. A direct Browser control now +keeps requested 256/320 kbps below an explicit 192 kbps receive limit, while a +510 kbps receiver permits both rates. These are local runtime checks; impaired +networks, mobile playback and physical listening remain open. + +RFC 7587 gives an omitted receive bitrate a mode-dependent default, not an +unrestricted value, and asks receivers to ignore out-of-range preferences. +All candidate Piik receivers therefore declare the full Opus range explicitly. +Browser sender enforcement uses [libwebrtc's existing default calculation](https://webrtc.googlesource.com/src/+/refs/heads/main/modules/audio_coding/codecs/opus/audio_encoder_opus.cc) +when a peer omits a valid limit. Unchanged forwarding requires an explicit +sufficient limit; it cannot turn down the shared source for one receiver. ## Primary Sources diff --git a/docs/research/media-fidelity.md b/docs/research/media-fidelity.md index 2f95f1a99..34f2c67b0 100644 --- a/docs/research/media-fidelity.md +++ b/docs/research/media-fidelity.md @@ -9,8 +9,8 @@ all-device acceptance or full HDR support. Browser display capture requests two channels without speech processing. The Host mixer uses a stereo Web Audio destination; Native capture and mixing use 48 kHz, two-channel PCM16. Native Opus uses the audio application mode. Browser, -Native and SFU receivers request stereo. The existing 64/128/192 kbps choices -are ceilings, not a lossless-audio promise. +Native and SFU receivers request stereo. Audio settings are bitrate ceilings, +not a lossless-audio promise; the media-quality owner defines the current controls. `opus/48000/2` alone does not prove stereo content: Opus permits mono packets under that declaration. [RFC 7587](https://www.rfc-editor.org/rfc/rfc7587.html#section-7) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 7beb9cde6..adffac81f 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -604,13 +604,14 @@ also travel in authentication and Native commands: adding its two enum values causes the actual v1.9.2 readers to terminate those sessions. The PR's compatible release claim is therefore incorrect even though its audio use case is valid. -For 2.0, prototype one numeric audio-bitrate ceiling through capture, sender, +The 2.0 candidate uses one numeric audio-bitrate ceiling through capture, sender, receiver and SFU; derive existing preset labels locally. Align receive admission with codec support instead of the largest UI shortcut. Keep the existing default while comparing actual Opus output and listening results for higher ceilings. The [numeric audio controls](./browser-screen-audio-quality.md#numeric-bitrate-assessment) establish local 256/320 kbps encoding and Browser reception, with separate -receive-limit enforcement and high-rate packet-size boundaries still to resolve. +receive-limit enforcement and a 320 kbps implementation ceiling. Full-range +Opus wire validation remains independent of that current packet-size choice. [LiveKit's AudioPreset](https://docs.livekit.io/reference/client-sdk-js/interfaces/AudioPreset.html) likewise contains a numeric maximum bitrate; its [high-fidelity example](https://docs.livekit.io/transport/media/advanced/#hi-fi-audio) diff --git a/docs/standards/media-quality.md b/docs/standards/media-quality.md index d8d81387d..980d0b8ec 100644 --- a/docs/standards/media-quality.md +++ b/docs/standards/media-quality.md @@ -336,10 +336,22 @@ application-defined whole-room lowest-common-denominator target. ## Screen Audio -Display audio uses `contentHint = "music"`. The live sender ceilings are 64, -128, and 192 kbps with 128 kbps default. Browser and Native Peer answers request -Opus stereo and a 192 kbps receive ceiling, including renegotiation. These are -local receive preferences, independent of the offer's receive preferences. +Display audio uses `contentHint = "music"`. The 2.0 candidate keeps 64/128/192 kbps +shortcuts and the 128 kbps default, with a custom numeric ceiling in the same +sound settings. Current Browser/native senders support 6..320 kbps; the numeric +wire format uses Opus's 6..510 kbps bounds independently of those shortcuts. +The current native 20 ms sender ceiling leaves packet-size headroom; a larger +codec-valid request fails that operation without retiring native control. +An audio-only update retains capture, media connections and video startup state. + +Browser, Native and SFU receive answers request Opus stereo and the full Opus +receive range, including renegotiation. Browser senders enforce a smaller +negotiated receive limit on their own encoding, falling back to libwebrtc's +mode-dependent default when no valid limit is declared. Native/SFU unchanged forwarding +requires a receiver that can accept the shared source's full range; a smaller +or undeclared limit rejects that edge before binding, without reducing the source or +its healthy siblings. An omitted preference does not mean unrestricted reception. +These receive preferences are independent of the offer's receive preferences. The SFU publication uses stereo, DTX disabled, and RED disabled; this accepts less burst-loss resilience in exchange for bounded publisher traffic. The Browser and source still decide whether an audio track diff --git a/docs/status.md b/docs/status.md index 5e3507f31..c99d58de8 100644 --- a/docs/status.md +++ b/docs/status.md @@ -8,7 +8,7 @@ and Git/PRs own completed history. ## Accepted Release Contract -- Browser/server v23, Native control v9 and capture v7 form the current contract. +- Browser/server v23, Native control v9 and capture v7 form the published contract. [Versioning](./standards/versioning.md#public-compatibility-promise) owns compatibility between published Web/App/Server versions. - Hosted Server defaults to SQLite schema 2. Room authority has no inactivity @@ -60,8 +60,12 @@ guidance preserve indefinite retention by default. The accepted Demo policy is one hour without authenticated participants; the private deployment retains the default. Operator records own activation, database and postflight evidence. The owner is holding the unpublished maintenance changes for 2.0. Preparation -includes separating media presets from numeric requests and bounded capabilities, -with audio presets, 4K and additional codecs assessed through that boundary. +includes separating media presets from numeric requests and bounded capabilities. +Numeric audio is implemented in the unpublished candidate: Browser/server v24 +and Native control v10 replace the preset field; capture v7 and SQLite schema 2 +remain unchanged. Matched candidate App/Web/Server builds are required; +1.x participants are rejected by the existing compatibility checks. +Geometry, 4K and additional codecs are still assessed through the media boundary. Native Magicsock remains an isolated experiment with incomplete product integration and unresolved recovery cases. These proposals do not yet change the accepted contract or transport. A small external test is proposed before diff --git a/docs/todo.md b/docs/todo.md index b12a638dd..68e04f187 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -47,19 +47,21 @@ Implementation needs a concrete useful result within the product model. - [ ] **Media parameters and capabilities for 2.0.** Follow the [media assessment](./research/native-client-media.md#media-capability-extension-assessment): separate UI presets, numeric requests, bounded implementation capabilities and - actual output. Complete numeric audio bitrate admission and storage first using - [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s use case, then + actual output. Numeric audio admission, storage and the compact custom input + implement [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s use case, with + local Browser/direct/relay and Browser/native SFU evidence. Next assess source-aspect-preserving geometry and 4K through the same owners. Preserve safety limits and framework-owned adaptation. The [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment) - require explicit receive-limit enforcement and a packet-size decision before - exposing higher numeric ceilings; add live relay/SFU acceptance to the local - encoding/Browser evidence. New codecs need complete + retain explicit receive-limit enforcement and the current 320 kbps sender + ceiling; physical listening and impaired-network evidence remain open. + New codecs need complete encode/decode/forward support and a late-Viewer policy before inclusion; selecting a codec cannot transcode an existing publication. Confirm live changes, relay/SFU paths, older 2.x capability subsets and explicit 1.x - mismatch before accepting the replacement contract. The current 1.x enums - remain; the PR is not a compatible enum-only extension of v1.9.2. + mismatch before accepting the replacement contract. The published 1.x enums + remain that major's baseline; the PR is not a compatible enum-only extension + of v1.9.2. - [ ] **Native Magicsock.** Go reuse is feasible, but the [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) have not established improved direct reachability. Seek a repeatable benefit diff --git a/internal/app/app.go b/internal/app/app.go index 9535d5b07..63b748e07 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -20,6 +20,7 @@ import ( "github.com/TNTcraftHIM/Piik/internal/app/lan" "github.com/TNTcraftHIM/Piik/internal/app/launcher" "github.com/TNTcraftHIM/Piik/internal/app/loopback" + "github.com/TNTcraftHIM/Piik/internal/app/nativeaudio" "github.com/TNTcraftHIM/Piik/internal/app/nativecapture" "github.com/TNTcraftHIM/Piik/internal/app/nativecontrol" "github.com/TNTcraftHIM/Piik/internal/app/publictunnel" @@ -626,6 +627,7 @@ func discoverNativeMedia(ctx context.Context, configuredPath string) nativeRunti captureProcess: path, capture: capabilities, capabilities: loopback.NativeMediaCapabilities{ + MaxAudioBitrate: nativeaudio.MaxSendBitrate, ReceiverReuse: true, Video: summary.Video, CaptureBorderControl: summary.CaptureBorderControl, diff --git a/internal/app/config/quality_preference_test.go b/internal/app/config/quality_preference_test.go index 10527de3b..a9c6dcb8f 100644 --- a/internal/app/config/quality_preference_test.go +++ b/internal/app/config/quality_preference_test.go @@ -20,7 +20,7 @@ func TestQualityPreferencePersistsWithoutChangingAppConfiguration(t *testing.T) t.Fatalf("initial preference: %v, %v", p.Read(), err) } quality := protocol.DefaultQualitySettings - quality.Resolution, quality.ScreenAudioQuality = "1440p", "very-high" + quality.Resolution, quality.MaxAudioBitrate = "1440p", 320_000 if err := p.Save(quality); err != nil { t.Fatal(err) } diff --git a/internal/app/loopback/protocol.go b/internal/app/loopback/protocol.go index ef7c0423d..8bd148095 100644 --- a/internal/app/loopback/protocol.go +++ b/internal/app/loopback/protocol.go @@ -9,9 +9,9 @@ import ( ) const ( - ProtocolVersion = 9 + ProtocolVersion = 10 ServiceName = "piik-client" - ControlSubprotocol = "piik-client-v9" + ControlSubprotocol = "piik-client-v10" MaxControlMessageBytes = 256 << 10 DefaultPortStart = 39721 DefaultPortEnd = 39730 diff --git a/internal/app/loopback/protocol_test.go b/internal/app/loopback/protocol_test.go index 5caf5ca00..9d354602a 100644 --- a/internal/app/loopback/protocol_test.go +++ b/internal/app/loopback/protocol_test.go @@ -7,8 +7,8 @@ import ( func TestDecodeRequestAcceptsOnlyTheCurrentShape(t *testing.T) { valid := []string{ - `{"version":9,"id":"request_hello","type":"hello"}`, - `{"version":9,"id":"request_ping","type":"ping"}`, + `{"version":10,"id":"request_hello","type":"hello"}`, + `{"version":10,"id":"request_ping","type":"ping"}`, } for _, payload := range valid { if _, err := decodeRequest([]byte(payload)); err != nil { @@ -17,10 +17,10 @@ func TestDecodeRequestAcceptsOnlyTheCurrentShape(t *testing.T) { } invalid := []string{ `{"version":3,"id":"request_ping","type":"ping"}`, - `{"version":9,"id":"short","type":"ping"}`, - `{"version":9,"id":"request_ping","type":"ping","extra":true}`, - `{"version":9,"id":"request_hello","type":"hello","nonce":"obsolete"}`, - `{"version":9,"id":"request_ping","type":"ping"} trailing`, + `{"version":10,"id":"short","type":"ping"}`, + `{"version":10,"id":"request_ping","type":"ping","extra":true}`, + `{"version":10,"id":"request_hello","type":"hello","nonce":"obsolete"}`, + `{"version":10,"id":"request_ping","type":"ping"} trailing`, } for _, payload := range invalid { if _, err := decodeRequest([]byte(payload)); err == nil { @@ -34,11 +34,11 @@ func TestDecodeRequestUsesTheSharedIdentifierBoundary(t *testing.T) { if len(identifier) != 256 { t.Fatalf("test identifier length = %d", len(identifier)) } - payload := []byte(`{"version":9,"id":"` + identifier + `","type":"ping"}`) + payload := []byte(`{"version":10,"id":"` + identifier + `","type":"ping"}`) if _, err := decodeRequest(payload); err != nil { t.Fatalf("maximum identifier was rejected: %v", err) } - tooLong := []byte(`{"version":9,"id":"` + identifier + `a","type":"ping"}`) + tooLong := []byte(`{"version":10,"id":"` + identifier + `a","type":"ping"}`) if _, err := decodeRequest(tooLong); err == nil { t.Fatal("identifier beyond the boundary was accepted") } @@ -62,7 +62,7 @@ func TestValidateMessagesRequireTheExpectedPhase(t *testing.T) { } func TestEnvelopeAllowsAnExtensionToOwnItsStrictShape(t *testing.T) { - payload := []byte(`{"version":9,"id":"request_extension","type":"extension","value":1}`) + payload := []byte(`{"version":10,"id":"request_extension","type":"extension","value":1}`) message, err := decodeEnvelope(payload) if err != nil || message.Type != "extension" { t.Fatalf("extension envelope = %+v, %v", message, err) diff --git a/internal/app/loopback/server.go b/internal/app/loopback/server.go index 8efc580d0..5a888f30e 100644 --- a/internal/app/loopback/server.go +++ b/internal/app/loopback/server.go @@ -72,6 +72,7 @@ type Health struct { } type NativeMediaCapabilities struct { + MaxAudioBitrate int `json:"maxAudioBitrate,omitempty"` ReceiverReuse bool `json:"receiverReuse,omitempty"` Video bool `json:"video"` CaptureBorderControl bool `json:"captureBorderControl,omitempty"` diff --git a/internal/app/mediaedge/edge.go b/internal/app/mediaedge/edge.go index 53580a4d3..16bd3dc24 100644 --- a/internal/app/mediaedge/edge.go +++ b/internal/app/mediaedge/edge.go @@ -8,6 +8,7 @@ import ( "strings" "sync" + "github.com/TNTcraftHIM/Piik/internal/app/nativeaudio" "github.com/TNTcraftHIM/Piik/internal/diagnostics" "github.com/TNTcraftHIM/Piik/internal/media/forwarding" "github.com/pion/ice/v4" @@ -256,6 +257,11 @@ func (edge *Edge) SetAnswer(answer webrtc.SessionDescription) error { return nil } edge.mu.Unlock() + if edge.audioSource != nil { + if err := forwarding.CheckAudioReceiveLimit(answer.SDP, nativeaudio.MaxBitrate); err != nil { + return err + } + } if err := edge.connection.SetRemoteDescription(answer); err != nil { return err } diff --git a/internal/app/mediaedge/edge_test.go b/internal/app/mediaedge/edge_test.go index d969bfea2..ac65c02b6 100644 --- a/internal/app/mediaedge/edge_test.go +++ b/internal/app/mediaedge/edge_test.go @@ -783,13 +783,11 @@ func connectEdgeToReceiver( t.Fatal(err) } gatherAnswer := webrtc.GatheringCompletePromise(receiver) - answer, err := receiver.CreateAnswer(nil) + // Use the product's receive preferences rather than Pion's generic defaults. + _, err := (&Receiver{connection: receiver}).createAnswer() if err != nil { t.Fatal(err) } - if err = receiver.SetLocalDescription(answer); err != nil { - t.Fatal(err) - } waitSignal(t, gatherAnswer, "answer ICE gathering") if err = edge.SetAnswer(*receiver.LocalDescription()); err != nil { t.Fatal(err) diff --git a/internal/app/mediaedge/publication.go b/internal/app/mediaedge/publication.go index c04127450..a860e08c6 100644 --- a/internal/app/mediaedge/publication.go +++ b/internal/app/mediaedge/publication.go @@ -84,6 +84,7 @@ func (engine *Engine) NewPublication(source *Source, options EdgeOptions) (*Publ publication.transport = transport transport.SetAudioBitrate(options.Audio.configuredBitrate()) publication.signaling.connection = transport.PC + publication.signaling.audioSource = options.Audio publication.signaling.localCandidates = newLocalCandidateGathering(socket, options.ICEServers, nil, options.Events.LocalCandidate) transport.PC.OnICECandidate(publication.signaling.localCandidates.addPion) transport.PC.OnConnectionStateChange(func(state webrtc.PeerConnectionState) { diff --git a/internal/app/mediaedge/receiver.go b/internal/app/mediaedge/receiver.go index d9810593d..4ca2a18f9 100644 --- a/internal/app/mediaedge/receiver.go +++ b/internal/app/mediaedge/receiver.go @@ -9,6 +9,7 @@ import ( "strings" "sync" + "github.com/TNTcraftHIM/Piik/internal/app/nativeaudio" "github.com/TNTcraftHIM/Piik/internal/media/encoded" "github.com/pion/rtcp" "github.com/pion/sdp/v3" @@ -249,7 +250,7 @@ func (receiver *Receiver) createAnswer() (webrtc.SessionDescription, error) { if !strings.EqualFold(codec.MimeType, webrtc.MimeTypeOpus) { continue } - codec.SDPFmtpLine = opusSDPFmtpLine + ";stereo=1;maxaveragebitrate=192000" + codec.SDPFmtpLine = opusSDPFmtpLine + ";stereo=1;maxaveragebitrate=" + strconv.Itoa(nativeaudio.MaxBitrate) if err := transceiver.SetCodecPreferences([]webrtc.RTPCodecParameters{codec}); err != nil { return webrtc.SessionDescription{}, err } diff --git a/internal/app/mediaedge/receiver_test.go b/internal/app/mediaedge/receiver_test.go index 0c6b1b394..af8ecc8dd 100644 --- a/internal/app/mediaedge/receiver_test.go +++ b/internal/app/mediaedge/receiver_test.go @@ -108,7 +108,7 @@ func TestReceiverAdvertisesLocalStereoPreference(t *testing.T) { continue } for _, attribute := range media.Attributes { - if attribute.Key == "fmtp" && strings.Contains(attribute.Value, "stereo=1") && strings.Contains(attribute.Value, "maxaveragebitrate=192000") { + if attribute.Key == "fmtp" && strings.Contains(attribute.Value, "stereo=1") && strings.Contains(attribute.Value, "maxaveragebitrate=510000") { found = true } } diff --git a/internal/app/nativeaudio/encoder.go b/internal/app/nativeaudio/encoder.go index d037d088c..08fad30df 100644 --- a/internal/app/nativeaudio/encoder.go +++ b/internal/app/nativeaudio/encoder.go @@ -8,12 +8,16 @@ import ( ) const ( - SampleRate = 48_000 - Channels = 2 - FrameSamples = 960 - FrameBytes = FrameSamples * Channels * 2 - MaxPacketBytes = 4_000 - DefaultBitrate = 128_000 + SampleRate = 48_000 + Channels = 2 + FrameSamples = 960 + FrameBytes = FrameSamples * Channels * 2 + MaxPacketBytes = 4_000 + DefaultBitrate = 128_000 + MinBitrate = 6_000 + MaxBitrate = 510_000 + // The 20 ms sender leaves RTP/SRTP headroom within an ordinary 1200-byte packet. + MaxSendBitrate = 320_000 defaultComplexity = 5 ) @@ -29,14 +33,14 @@ type Encoder struct { func (encoder *Encoder) Bitrate() int { return encoder.codec.Bitrate() } func (encoder *Encoder) SetBitrate(bitrate int) error { - if encoder == nil || encoder.codec == nil || bitrate < 6_000 || bitrate > 510_000 { + if encoder == nil || encoder.codec == nil || bitrate < MinBitrate || bitrate > MaxBitrate { return errors.New("native audio bitrate is outside the Opus range") } return encoder.codec.SetBitrate(bitrate) } func NewEncoder(bitrate int) (*Encoder, error) { - if bitrate < 6_000 || bitrate > 510_000 { + if bitrate < MinBitrate || bitrate > MaxBitrate { return nil, errors.New("native audio bitrate is outside the Opus range") } codec, err := gopus.NewEncoder(gopus.EncoderConfig{ diff --git a/internal/app/nativecontrol/operation_failure_test.go b/internal/app/nativecontrol/operation_failure_test.go index 490f88cae..4a69ba5bd 100644 --- a/internal/app/nativecontrol/operation_failure_test.go +++ b/internal/app/nativecontrol/operation_failure_test.go @@ -62,7 +62,7 @@ func TestSourceProbesRetireWithTheirControlConnection(t *testing.T) { t.Fatal(err) } defer connection.CloseNow() - if err = wsjson.Write(ctx, connection, json.RawMessage(`{"version":9,"id":"request_hello","type":"hello"}`)); err != nil { + if err = wsjson.Write(ctx, connection, json.RawMessage(`{"version":10,"id":"request_hello","type":"hello"}`)); err != nil { t.Fatal(err) } var ready struct{ Type string } @@ -78,7 +78,7 @@ func TestSourceProbesRetireWithTheirControlConnection(t *testing.T) { t.Error("native probe fixture did not retire") } }() - payload := fmt.Sprintf(`{"version":9,"id":"request_probe","type":%q%s}`, request.kind, request.fields) + payload := fmt.Sprintf(`{"version":10,"id":"request_probe","type":%q%s}`, request.kind, request.fields) if err = wsjson.Write(ctx, connection, json.RawMessage(payload)); err != nil { t.Fatal(err) } @@ -150,7 +150,7 @@ func TestOperationalFailureAndIdempotentStopKeepTheRealControlConnection(t *test {"ping", "", "pong"}, } { id := fmt.Sprintf("request_%d", index) - payload := fmt.Sprintf(`{"version":9,"id":%q,"type":%q%s}`, id, command.kind, command.fields) + payload := fmt.Sprintf(`{"version":10,"id":%q,"type":%q%s}`, id, command.kind, command.fields) if err := wsjson.Write(ctx, connection, json.RawMessage(payload)); err != nil { t.Fatal(err) } @@ -164,7 +164,7 @@ func TestOperationalFailureAndIdempotentStopKeepTheRealControlConnection(t *test } } // A gone target must not bypass strict publication validation. - if err := wsjson.Write(ctx, connection, json.RawMessage(`{"version":9,"id":"request_invalid","type":"close-publication","shareId":"missing_share","publicationGeneration":"publication_123","connectionId":"edge_123456","extra":true}`)); err != nil { + if err := wsjson.Write(ctx, connection, json.RawMessage(`{"version":10,"id":"request_invalid","type":"close-publication","shareId":"missing_share","publicationGeneration":"publication_123","connectionId":"edge_123456","extra":true}`)); err != nil { t.Fatal(err) } if _, _, err := connection.Read(ctx); websocket.CloseStatus(err) != websocket.StatusPolicyViolation { @@ -180,7 +180,7 @@ func TestFailedEdgeAndLateStopDoNotRetireAnotherShare(t *testing.T) { t.Setenv("PIIK_QUIET_CAPTURE_FIXTURE", t.TempDir()) session := New(executable, nativecapture.Capabilities{SoftwareVP8: true}, false) defer session.Close() - const start = `{"version":9,"id":"request_start","type":"start-share","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"audio":false,"edgeCapacity":1,"codec":"vp8","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}}` + const start = `{"version":10,"id":"request_start","type":"start-share","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"audio":false,"edgeCapacity":1,"codec":"vp8","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}}` if value, err := awaitControlResponse(t, session, []byte(start)); err != nil { t.Fatal(err) } else if _, ok := value.(shareStartedResponse); !ok { @@ -190,14 +190,14 @@ func TestFailedEdgeAndLateStopDoNotRetireAnotherShare(t *testing.T) { if host == nil { t.Fatal("share was not installed") } - const edge = `{"version":9,"id":"request_edge","type":"prepare-local-edge","shareId":"share_123456","connectionId":"edge_123456"}` + const edge = `{"version":10,"id":"request_edge","type":"prepare-local-edge","shareId":"share_123456","connectionId":"edge_123456"}` if value, err := session.Handle(t.Context(), []byte(edge)); err != nil { t.Fatal(err) } else if _, ok := value.(edgeOfferResponse); !ok { t.Fatalf("edge preparation failed: %#v", value) } assertOperationFailure(t, session, edge) - assertOperationFailure(t, session, `{"version":9,"id":"request_edge","type":"prepare-edge","shareId":"stale_share","connectionId":"edge_123456","iceServers":[]}`) + assertOperationFailure(t, session, `{"version":10,"id":"request_edge","type":"prepare-edge","shareId":"stale_share","connectionId":"edge_123456","iceServers":[]}`) for _, text := range []string{strings.Repeat("界", 4096), strings.Repeat("界", 4097)} { candidate, _ := json.Marshal(map[string]any{"candidate": text}) var upstream protocol.IceCandidate @@ -209,7 +209,7 @@ func TestFailedEdgeAndLateStopDoNotRetireAnotherShare(t *testing.T) { if kind == "publication-candidate" { fields = `,"publicationGeneration":"publication_123"` } - payload := fmt.Sprintf(`{"version":9,"id":"request_remote","type":%q,"shareId":"share_123456","connectionId":"edge_123456","candidate":%s%s}`, kind, candidate, fields) + payload := fmt.Sprintf(`{"version":10,"id":"request_remote","type":%q,"shareId":"share_123456","connectionId":"edge_123456","candidate":%s%s}`, kind, candidate, fields) if len([]rune(text)) > 4096 { assertOperationFailure(t, session, payload) } else if _, err := session.Handle(t.Context(), []byte(payload)); err != nil { @@ -233,8 +233,8 @@ func TestFailedEdgeAndLateStopDoNotRetireAnotherShare(t *testing.T) { t.Fatal(err) } for range 2 { - value, err := session.Handle(t.Context(), []byte(`{"version":9,"id":"request_stop","type":"stop-share","shareId":"share_123456"}`)) - if err != nil || value != response(requestEnvelope{Version: 9, ID: "request_stop"}, "share-stopped") { + value, err := session.Handle(t.Context(), []byte(`{"version":10,"id":"request_stop","type":"stop-share","shareId":"share_123456"}`)) + if err != nil || value != response(requestEnvelope{Version: 10, ID: "request_stop"}, "share-stopped") { t.Fatalf("late stop: %#v, %v", value, err) } } @@ -274,7 +274,7 @@ func TestCaptureStartupAndReplacementRemainCancellable(t *testing.T) { } session := New(executable, nativecapture.Capabilities{SoftwareVP8: true}, false) defer session.Close() - start := `{"version":9,"id":"request_start","type":"start-share","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"edgeCapacity":1,"codec":"vp8","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}}` + start := `{"version":10,"id":"request_start","type":"start-share","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"edgeCapacity":1,"codec":"vp8","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}}` payload := strings.Replace(start, "1080p", "720p", 1) if kind == "replace-share-source" { if value, err := awaitControlResponse(t, session, []byte(start)); err != nil { @@ -282,7 +282,7 @@ func TestCaptureStartupAndReplacementRemainCancellable(t *testing.T) { } else if _, ok := value.(shareStartedResponse); !ok { t.Fatalf("start: %#v", value) } - payload = `{"version":9,"id":"request_replace","type":"replace-share-source","shareId":"share_123456","source":{"kind":"display","sourceId":"2","title":"Fixture"},"audio":false}` + payload = `{"version":10,"id":"request_replace","type":"replace-share-source","shareId":"share_123456","source":{"kind":"display","sourceId":"2","title":"Fixture"},"audio":false}` } t.Setenv("PIIK_PENDING_CAPTURE_FIXTURE", "true") if value, err := session.Handle(t.Context(), []byte(payload)); value != nil || err != nil { @@ -300,12 +300,12 @@ func TestCaptureStartupAndReplacementRemainCancellable(t *testing.T) { } // No unbounded parallel initialization while the first is still waiting. assertOperationFailure(t, session, payload) - if _, err := session.Handle(t.Context(), []byte(`{"version":9,"id":"request_ice","type":"edge-candidate","shareId":"share_123456","connectionId":"missing_edge","candidate":null}`)); err != nil { + if _, err := session.Handle(t.Context(), []byte(`{"version":10,"id":"request_ice","type":"edge-candidate","shareId":"share_123456","connectionId":"missing_edge","candidate":null}`)); err != nil { t.Fatalf("capture initialization blocked independent control: %v", err) } stopped := make(chan error, 1) go func() { - _, err := session.Handle(t.Context(), []byte(`{"version":9,"id":"request_stop","type":"stop-share","shareId":"share_123456"}`)) + _, err := session.Handle(t.Context(), []byte(`{"version":10,"id":"request_stop","type":"stop-share","shareId":"share_123456"}`)) stopped <- err }() select { diff --git a/internal/app/nativecontrol/session.go b/internal/app/nativecontrol/session.go index 40a59a71f..f3e8e3ab6 100644 --- a/internal/app/nativecontrol/session.go +++ b/internal/app/nativecontrol/session.go @@ -178,10 +178,12 @@ func (session *Session) Handle(ctx context.Context, payload []byte) (result any, !identityPattern.MatchString(request.ShareID) || !request.Source.Valid() || request.EdgeCapacity < 1 || request.EdgeCapacity > maxEdgeCapacity || - (request.Codec != "auto" && request.Codec != "h264" && request.Codec != "vp8") || - !validQualitySettings(request.Profile) { + (request.Codec != "auto" && request.Codec != "h264" && request.Codec != "vp8") { return nil, protocolViolation("native start-share request is invalid") } + if !nativeQualityProfile(request.Profile).Valid() { + return nil, errors.New("native sharing profile is unsupported") + } if err := session.validateAudioExclusion(request.Source, request.Audio, request.ExcludeAudio); err != nil { return nil, err } @@ -226,11 +228,13 @@ func (session *Session) Handle(ctx context.Context, payload []byte) (result any, var request updateShareRequest if err := decodeStrict(payload, &request); err != nil || request.Type != envelope.Type || - !validIdentities(request.ShareID) || - !validQualitySettings(request.Profile) { + !validIdentities(request.ShareID) { return nil, protocolViolation("native update-share request is invalid") } profile := nativeQualityProfile(request.Profile) + if !profile.Valid() { + return nil, errors.New("native sharing profile is unsupported") + } slog.Debug("piik-client", "event", "native-profile-requested", "requestId", envelope.ID, "share", diagnostics.ID(request.ShareID), "profile", profile) var err error @@ -636,10 +640,6 @@ func (session *Session) startShare( }, nil } -func validQualitySettings(settings qualitySettings) bool { - return nativeQualityProfile(settings).Valid() -} - func nativeQualityProfile(settings qualitySettings) nativehost.QualityProfile { width, height := uint32(0), uint32(0) switch settings.Resolution { @@ -652,21 +652,16 @@ func nativeQualityProfile(settings qualitySettings) nativehost.QualityProfile { case "1440p": width, height = 2560, 1440 } - audioBitrate := 0 - switch settings.ScreenAudioQuality { - case "", "music": - audioBitrate = 128_000 - case "saver": - audioBitrate = 64_000 - case "very-high": - audioBitrate = 192_000 + audioBitrate := int(settings.MaxAudioBitrate) + if audioBitrate == 0 { + audioBitrate = int(protocol.DefaultQualitySettings.MaxAudioBitrate) } return nativehost.QualityProfile{ Video: nativecapture.VideoProfile{ Width: width, Height: height, - Framerate: settings.MaxFramerate, - Bitrate: settings.MaxBitrate, + Framerate: uint32(settings.MaxFramerate), + Bitrate: uint32(settings.MaxBitrate), Preference: settings.DegradationPreference, }, AudioBitrate: audioBitrate, diff --git a/internal/app/nativecontrol/session_test.go b/internal/app/nativecontrol/session_test.go index d626d45c2..de572e1e8 100644 --- a/internal/app/nativecontrol/session_test.go +++ b/internal/app/nativecontrol/session_test.go @@ -67,7 +67,7 @@ func TestReceiveOfferReusePreservesLegacyResponseShape(t *testing.T) { session := New("", nativecapture.Capabilities{}, false) t.Cleanup(func() { _ = session.Close() }) for _, reuse := range []bool{false, true} { - request := receiveOfferRequest{Version: 9, ID: "request_offer", Type: "receive-offer", ShareID: "viewer_session", + request := receiveOfferRequest{Version: 10, ID: "request_offer", Type: "receive-offer", ShareID: "viewer_session", ConnectionID: "connection_1234", EdgeCapacity: 2, SDP: offer.SDP, ReuseReceiver: reuse} payload, _ := json.Marshal(request) value, err := session.Handle(t.Context(), payload) @@ -142,7 +142,7 @@ func TestSourceListFitsTheControlEnvelope(t *testing.T) { session := New(executable, nativecapture.Capabilities{}, false) defer session.Close() value, err := awaitControlResponse(t, session, - []byte(`{"version":9,"id":"request_sources","type":"list-sources"}`)) + []byte(`{"version":10,"id":"request_sources","type":"list-sources"}`)) if err != nil { t.Fatal(err) } @@ -250,7 +250,7 @@ func TestMicrophoneOptInPreservesLegacyAndOrdersStateBeforeCompletion(t *testing } session := New(executable, nativecapture.Capabilities{VideoCapture: true, SoftwareVP8: true, Microphone: true}, false) defer session.Close() - request := startShareRequest{Version: 9, ID: "request_start", Type: "start-share", ShareID: "share_123456", + request := startShareRequest{Version: 10, ID: "request_start", Type: "start-share", ShareID: "share_123456", Source: nativecapture.CaptureTarget{Kind: "display", SourceID: "1", Title: "Fixture"}, MicrophoneMixing: optIn, EdgeCapacity: 1, Codec: "vp8", Profile: qualitySettings{Resolution: "1080p", MaxFramerate: 30, MaxBitrate: 5_000_000, DegradationPreference: "balanced"}} @@ -264,7 +264,7 @@ func TestMicrophoneOptInPreservesLegacyAndOrdersStateBeforeCompletion(t *testing if started.Audio != optIn || bytes.Contains(encoded, []byte(`"sourceAudio"`)) != optIn { t.Fatalf("mixed output leaked across opt-in: %s", encoded) } - result, err = session.Handle(t.Context(), []byte(`{"version":9,"id":"request_mic","type":"set-microphone","shareId":"share_123456","enabled":true,"volume":1}`)) + result, err = session.Handle(t.Context(), []byte(`{"version":10,"id":"request_mic","type":"set-microphone","shareId":"share_123456","enabled":true,"volume":1}`)) if !optIn { if _, failed := result.(requestFailedResponse); err != nil || !failed { t.Fatalf("non-opted share admitted mic: %v %#v", err, result) @@ -332,7 +332,7 @@ func TestCaptureBorderPreferenceFollowsSourceAndProfile(t *testing.T) { t.Fatalf("capture border intent lost: %v", arguments) } } - start := startShareRequest{Version: 9, ID: "request_start", Type: "start-share", ShareID: "share_123456", + start := startShareRequest{Version: 10, ID: "request_start", Type: "start-share", ShareID: "share_123456", Source: nativecapture.CaptureTarget{Kind: "display", SourceID: "1", Title: "Fixture"}, ShowCaptureBorder: true, EdgeCapacity: 1, Codec: "vp8", Profile: qualitySettings{Resolution: "1080p", MaxFramerate: 30, MaxBitrate: 5_000_000, DegradationPreference: "balanced"}} @@ -340,7 +340,7 @@ func TestCaptureBorderPreferenceFollowsSourceAndProfile(t *testing.T) { assertBorder(supported) profile := start.Profile profile.Resolution = "480p" - handle(updateShareRequest{Version: 9, ID: "request_update", Type: "update-share", ShareID: start.ShareID, Profile: profile}) + handle(updateShareRequest{Version: 10, ID: "request_update", Type: "update-share", ShareID: start.ShareID, Profile: profile}) session.mu.Lock() done := session.updateDone session.mu.Unlock() @@ -353,11 +353,11 @@ func TestCaptureBorderPreferenceFollowsSourceAndProfile(t *testing.T) { } assertBorder(supported) for _, show := range []bool{false, true} { - handle(replaceShareSourceRequest{Version: 9, ID: "request_replace", Type: "replace-share-source", ShareID: start.ShareID, + handle(replaceShareSourceRequest{Version: 10, ID: "request_replace", Type: "replace-share-source", ShareID: start.ShareID, Source: nativecapture.CaptureTarget{Kind: "display", SourceID: "2", Title: "Other fixture"}, ShowCaptureBorder: show}) assertBorder(show && supported) } - handle(stopShareRequest{Version: 9, ID: "request_stop", Type: "stop-share", ShareID: start.ShareID}) + handle(stopShareRequest{Version: 10, ID: "request_stop", Type: "stop-share", ShareID: start.ShareID}) }) } } @@ -393,9 +393,9 @@ func TestQuietHostUpdatePreservesControlAndCancelsCleanly(t *testing.T) { } return value } - const start = `{"version":9,"id":"request_start","type":"start-share","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"audio":false,"adapterIndex":0,"encoderIndex":0,"edgeCapacity":1,"codec":"vp8","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}}` + const start = `{"version":10,"id":"request_start","type":"start-share","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"audio":false,"adapterIndex":0,"encoderIndex":0,"edgeCapacity":1,"codec":"vp8","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}}` handle(start) - const update = `{"version":9,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"720p","maxFramerate":30,"maxBitrate":3000000,"degradationPreference":"balanced"}}` + const update = `{"version":10,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"720p","maxFramerate":30,"maxBitrate":3000000,"degradationPreference":"balanced"}}` if value := handle(update); value != nil { t.Fatalf("Host update did not defer: %#v", value) } @@ -414,15 +414,15 @@ func TestQuietHostUpdatePreservesControlAndCancelsCleanly(t *testing.T) { } for _, payload := range []string{ strings.Replace(update, "request_update", "request_duplicate", 1), - `{"version":9,"id":"request_replace","type":"replace-share-source","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"audio":false,"adapterIndex":0,"encoderIndex":0}`, + `{"version":10,"id":"request_replace","type":"replace-share-source","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"audio":false,"adapterIndex":0,"encoderIndex":0}`, } { if _, ok := handle(payload).(requestFailedResponse); !ok { t.Fatal("conflicting capture mutation was not rejected") } } for _, payload := range []string{ - `{"version":9,"id":"request_pause","type":"pause-share","shareId":"share_123456","paused":true}`, - `{"version":9,"id":"request_ice","type":"edge-candidate","shareId":"share_123456","connectionId":"retired_edge","candidate":null}`, + `{"version":10,"id":"request_pause","type":"pause-share","shareId":"share_123456","paused":true}`, + `{"version":10,"id":"request_ice","type":"edge-candidate","shareId":"share_123456","connectionId":"retired_edge","candidate":null}`, } { started := time.Now() handle(payload) @@ -457,7 +457,7 @@ func TestQuietHostUpdatePreservesControlAndCancelsCleanly(t *testing.T) { t.Fatal(err) } case "stop": - handle(`{"version":9,"id":"request_stop","type":"stop-share","shareId":"share_123456"}`) + handle(`{"version":10,"id":"request_stop","type":"stop-share","shareId":"share_123456"}`) case "disconnect": if err = session.Close(); err != nil { t.Fatal(err) @@ -474,7 +474,7 @@ func TestQuietHostUpdatePreservesControlAndCancelsCleanly(t *testing.T) { if outcome == "backpressure-stop" { stopped := make(chan any, 1) go func() { - value, err := session.Handle(t.Context(), []byte(`{"version":9,"id":"request_stop","type":"stop-share","shareId":"share_123456"}`)) + value, err := session.Handle(t.Context(), []byte(`{"version":10,"id":"request_stop","type":"stop-share","shareId":"share_123456"}`)) if err != nil { stopped <- err } else { @@ -577,7 +577,7 @@ func TestControlMessagesRejectUnknownFieldsAndStaleVersions(t *testing.T) { session := New("missing-capture-process", nativecapture.Capabilities{}, false) t.Cleanup(func() { _ = session.Close() }) for _, payload := range []string{ - `{"version":9,"id":"request_sources","type":"list-sources","extra":true}`, + `{"version":10,"id":"request_sources","type":"list-sources","extra":true}`, `{"version":4,"id":"request_sources","type":"list-sources"}`, `{"version":5,"id":"short","type":"stop-share","shareId":"share_123456"}`, } { @@ -609,9 +609,9 @@ func TestResponseKeepsTheRequestIdentity(t *testing.T) { func TestPrepareLocalEdgeOwnsOneStrictRequestShape(t *testing.T) { session := New("missing-capture-process", nativecapture.Capabilities{}, false) t.Cleanup(func() { _ = session.Close() }) - valid := `{"version":9,"id":"request_local_edge","type":"prepare-local-edge","shareId":"share_123456","connectionId":"edge_1234567"}` + valid := `{"version":10,"id":"request_local_edge","type":"prepare-local-edge","shareId":"share_123456","connectionId":"edge_1234567"}` assertOperationFailure(t, session, valid) - invalid := `{"version":9,"id":"request_local_edge","type":"prepare-local-edge","shareId":"share_123456","connectionId":"edge_1234567","iceServers":[]}` + invalid := `{"version":10,"id":"request_local_edge","type":"prepare-local-edge","shareId":"share_123456","connectionId":"edge_1234567","iceServers":[]}` if _, err := session.Handle(t.Context(), []byte(invalid)); err == nil || err.Error() != "native prepare-local-edge request is invalid" { t.Fatalf("extended local-edge request was accepted: %v", err) } @@ -621,7 +621,7 @@ func TestPreviewFailureReturnsAnAdvisoryResponse(t *testing.T) { session := New("missing-capture-process", nativecapture.Capabilities{}, false) t.Cleanup(func() { _ = session.Close() }) value, err := session.Handle(t.Context(), []byte( - `{"version":9,"id":"request_preview","type":"source-preview","source":{"kind":"display","sourceId":"65537","title":"Display 1"}}`, + `{"version":10,"id":"request_preview","type":"source-preview","source":{"kind":"display","sourceId":"65537","title":"Display 1"}}`, )) if err != nil { t.Fatal(err) @@ -636,7 +636,7 @@ func TestSourceEnumerationFailureDoesNotCloseControl(t *testing.T) { session := New("missing-capture-process", nativecapture.Capabilities{}, false) t.Cleanup(func() { _ = session.Close() }) value, err := session.Handle(t.Context(), []byte( - `{"version":9,"id":"request_sources","type":"list-sources"}`, + `{"version":10,"id":"request_sources","type":"list-sources"}`, )) if err != nil || value != operationFailure(requestEnvelope{ Version: loopback.ProtocolVersion, ID: "request_sources", Type: "list-sources", @@ -644,7 +644,7 @@ func TestSourceEnumerationFailureDoesNotCloseControl(t *testing.T) { t.Fatalf("source list failure ended its control session: %#v, %v", value, err) } if _, err := session.Handle(t.Context(), []byte( - `{"version":9,"id":"request_options","type":"capture-options"}`, + `{"version":10,"id":"request_options","type":"capture-options"}`, )); err != nil { t.Fatalf("control was unusable after source enumeration failed: %v", err) } @@ -654,10 +654,10 @@ func TestQualitySettingsMapOnceIntoNativeMedia(t *testing.T) { settings := qualitySettings{ Resolution: "1440p", MaxFramerate: 60, MaxBitrate: 12_000_000, DegradationPreference: "maintain-framerate", - ScreenAudioQuality: "very-high", + MaxAudioBitrate: 192000, } profile := nativeQualityProfile(settings) - if !validQualitySettings(settings) || profile.Video.Width != 2560 || + if !profile.Valid() || profile.Video.Width != 2560 || profile.Video.Height != 1440 || profile.Video.Framerate != 60 || profile.Video.Bitrate != 12_000_000 || profile.Video.Preference != "maintain-framerate" || @@ -665,7 +665,7 @@ func TestQualitySettingsMapOnceIntoNativeMedia(t *testing.T) { t.Fatalf("native profile = %+v", profile) } settings.Resolution = "2160p" - if validQualitySettings(settings) { + if nativeQualityProfile(settings).Valid() { t.Fatal("unsupported resolution was accepted") } } @@ -673,9 +673,15 @@ func TestQualitySettingsMapOnceIntoNativeMedia(t *testing.T) { func TestUpdateShareRequiresTheCurrentStrictProfile(t *testing.T) { session := New("missing-capture-process", nativecapture.Capabilities{}, false) t.Cleanup(func() { _ = session.Close() }) - valid := `{"version":9,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced","screenAudioQuality":"music"}}` + valid := `{"version":10,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced","maxAudioBitrate":128000}}` assertOperationFailure(t, session, valid) - invalid := `{"version":9,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced","screenAudioQuality":"music","extra":true}}` + // Opus permits this numeric request, but the current 20 ms sender does not. + // Keep the control session usable and let the caller keep its current share. + assertOperationFailure(t, session, strings.Replace(valid, `"maxAudioBitrate":128000`, `"maxAudioBitrate":510000`, 1)) + if _, err := session.Handle(t.Context(), []byte(`{"version":10,"id":"request_options","type":"capture-options"}`)); err != nil { + t.Fatalf("unsupported media setting retired control: %v", err) + } + invalid := `{"version":10,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced","maxAudioBitrate":128000,"extra":true}}` if _, err := session.Handle(t.Context(), []byte(invalid)); err == nil || err.Error() != "native update-share request is invalid" { t.Fatalf("extended update was accepted: %v", err) @@ -685,9 +691,9 @@ func TestUpdateShareRequiresTheCurrentStrictProfile(t *testing.T) { func TestReplaceShareSourceKeepsOneStrictTargetShape(t *testing.T) { session := New("missing-capture-process", nativecapture.Capabilities{}, false) t.Cleanup(func() { _ = session.Close() }) - valid := `{"version":9,"id":"request_source","type":"replace-share-source","shareId":"share_123456","source":{"kind":"picker","sourceId":"1","title":"Portal"},"audio":false,"adapterIndex":0,"encoderIndex":0}` + valid := `{"version":10,"id":"request_source","type":"replace-share-source","shareId":"share_123456","source":{"kind":"picker","sourceId":"1","title":"Portal"},"audio":false,"adapterIndex":0,"encoderIndex":0}` assertOperationFailure(t, session, valid) - invalid := `{"version":9,"id":"request_source","type":"replace-share-source","shareId":"share_123456","source":{"kind":"picker","sourceId":"1","title":"Portal"},"audio":false,"adapterIndex":0,"encoderIndex":0,"extra":true}` + invalid := `{"version":10,"id":"request_source","type":"replace-share-source","shareId":"share_123456","source":{"kind":"picker","sourceId":"1","title":"Portal"},"audio":false,"adapterIndex":0,"encoderIndex":0,"extra":true}` if _, err := session.Handle(t.Context(), []byte(invalid)); err == nil || err.Error() != "native replace-share-source request is invalid" { t.Fatalf("extended source replacement was accepted: %v", err) diff --git a/internal/app/nativecontrol/wire.go b/internal/app/nativecontrol/wire.go index b69ab1ca6..eddcda346 100644 --- a/internal/app/nativecontrol/wire.go +++ b/internal/app/nativecontrol/wire.go @@ -3,6 +3,7 @@ package nativecontrol import ( "github.com/TNTcraftHIM/Piik/internal/app/mediaedge" "github.com/TNTcraftHIM/Piik/internal/app/nativecapture" + "github.com/TNTcraftHIM/Piik/internal/server/protocol" "github.com/pion/webrtc/v4" ) @@ -33,13 +34,7 @@ type sourcePreviewRequest struct { Source nativecapture.CaptureTarget `json:"source"` } -type qualitySettings struct { - Resolution string `json:"resolution"` - MaxFramerate uint32 `json:"maxFramerate"` - MaxBitrate uint32 `json:"maxBitrate"` - DegradationPreference string `json:"degradationPreference"` - ScreenAudioQuality string `json:"screenAudioQuality,omitempty"` -} +type qualitySettings = protocol.QualitySettings type startShareRequest struct { Version int `json:"version"` diff --git a/internal/app/nativehost/profile.go b/internal/app/nativehost/profile.go index 7e02897da..90dd27011 100644 --- a/internal/app/nativehost/profile.go +++ b/internal/app/nativehost/profile.go @@ -1,6 +1,9 @@ package nativehost -import "github.com/TNTcraftHIM/Piik/internal/app/nativecapture" +import ( + "github.com/TNTcraftHIM/Piik/internal/app/nativeaudio" + "github.com/TNTcraftHIM/Piik/internal/app/nativecapture" +) type QualityProfile struct { Video nativecapture.VideoProfile @@ -9,7 +12,5 @@ type QualityProfile struct { func (profile QualityProfile) Valid() bool { return profile.Video.Valid() && - (profile.AudioBitrate == 64_000 || - profile.AudioBitrate == 128_000 || - profile.AudioBitrate == 192_000) + profile.AudioBitrate >= nativeaudio.MinBitrate && profile.AudioBitrate <= nativeaudio.MaxSendBitrate } diff --git a/internal/media/forwarding/audio.go b/internal/media/forwarding/audio.go new file mode 100644 index 000000000..787b36f6c --- /dev/null +++ b/internal/media/forwarding/audio.go @@ -0,0 +1,52 @@ +package forwarding + +import ( + "errors" + "strconv" + "strings" + + "github.com/pion/sdp/v3" +) + +// CheckAudioReceiveLimit admits unchanged Opus forwarding. A receiver may not +// lower the shared source or its siblings. Piik receivers declare the full Opus +// range; an omitted preference uses codec defaults, not an unrestricted rate. +func CheckAudioReceiveLimit(description string, sourceCeiling int) error { + var session sdp.SessionDescription + if err := session.Unmarshal([]byte(description)); err != nil { + return err + } + for _, media := range session.MediaDescriptions { + if media.MediaName.Media != "audio" || media.MediaName.Port.Value == 0 { + continue + } + for _, format := range media.MediaName.Formats { + payload, err := strconv.ParseUint(format, 10, 8) + if err != nil { + continue + } + codec, err := session.GetCodecForPayloadType(uint8(payload)) + if err != nil || !strings.EqualFold(codec.Name, "opus") { + continue + } + receiveLimit := 0 + for _, parameter := range strings.Split(codec.Fmtp, ";") { + name, value, found := strings.Cut(parameter, "=") + if !strings.EqualFold(strings.TrimSpace(name), "maxaveragebitrate") { + continue + } + limit, err := strconv.Atoi(strings.TrimSpace(value)) + if !found || err != nil || limit < 6_000 || limit > 510_000 { + continue // RFC 7587: ignore invalid preferences. + } + if receiveLimit == 0 || limit < receiveLimit { + receiveLimit = limit + } + } + if receiveLimit < sourceCeiling { + return errors.New("receiver audio bitrate limit cannot carry the shared Opus source") + } + } + } + return nil +} diff --git a/internal/media/forwarding/audio_test.go b/internal/media/forwarding/audio_test.go new file mode 100644 index 000000000..7c249d0d7 --- /dev/null +++ b/internal/media/forwarding/audio_test.go @@ -0,0 +1,34 @@ +package forwarding + +import ( + "strings" + "testing" +) + +func TestSharedAudioRejectsOnlyAnInsufficientReceiver(t *testing.T) { + for _, test := range []struct { + parameter string + allowed bool + }{ + {"stereo=1", false}, + {"stereo=1;maxaveragebitrate=510000", true}, + {"stereo=1;maxaveragebitrate=192000", false}, + {"maxaveragebitrate=320000", false}, + {"maxaveragebitrate=invalid", false}, + {"maxaveragebitrate=510001", false}, + {"maxaveragebitrate=510000;maxaveragebitrate=510001", true}, + {"maxaveragebitrate=510000;maxaveragebitrate=96000", false}, + } { + t.Run(test.parameter, func(t *testing.T) { + answer := "v=0\r\no=- 1 1 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\n" + + "m=audio 9 UDP/TLS/RTP/SAVPF 111\r\na=rtpmap:111 opus/48000/2\r\na=fmtp:111 " + test.parameter + "\r\n" + if err := CheckAudioReceiveLimit(answer, 510_000); (err == nil) != test.allowed { + t.Fatalf("receiver admitted=%v, err=%v", test.allowed, err) + } + // Rejected or unrelated media cannot impose an audio limit. + if err := CheckAudioReceiveLimit(strings.Replace(answer, "m=audio 9", "m=audio 0", 1), 510_000); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/server/protocol/constants.go b/internal/server/protocol/constants.go index d70fded9f..cb393307b 100644 --- a/internal/server/protocol/constants.go +++ b/internal/server/protocol/constants.go @@ -7,12 +7,18 @@ package protocol // MaxSafeInteger is JavaScript's Number.MAX_SAFE_INTEGER. const MaxSafeInteger = 9007199254740991 +// Opus format bounds, independent of UI presets and current sender capabilities. +const ( + MinAudioBitrate = 6_000 + MaxAudioBitrate = 510_000 +) + // Limits and identifiers in the Piik wire contract. const ( MaxViewersPerRoomLimit = 20 MaxParticipantsPerRoomLimit = MaxViewersPerRoomLimit + 1 MaxSignalBytes = 64 * 1024 - SignalingProtocol = "piik-v23" + SignalingProtocol = "piik-v24" RoomCodeLength = 4 MaxMediaRouteRevision = MaxSafeInteger MaxIceServerURLs = 8 @@ -62,7 +68,7 @@ var DefaultQualitySettings = QualitySettings{ MaxFramerate: 30, MaxBitrate: 5_000_000, DegradationPreference: "balanced", - ScreenAudioQuality: "music", + MaxAudioBitrate: 128000, } // DefaultRoutePolicy mirrors DEFAULT_ROUTE_POLICY. diff --git a/internal/server/protocol/contract_test.go b/internal/server/protocol/contract_test.go index 9e127ef29..206999b58 100644 --- a/internal/server/protocol/contract_test.go +++ b/internal/server/protocol/contract_test.go @@ -126,7 +126,7 @@ const ( func authenticatedHost() []member { return []member{ {"type", `"authenticated"`}, - {"protocol", `"piik-v23"`}, + {"protocol", `"piik-v24"`}, {"peerId", `"host_12345678"`}, {"maxViewers", `8`}, {"endpointMediaCopyCapacity", `2`}, @@ -705,7 +705,7 @@ func TestAuthenticateConnectionProgressOptIn(t *testing.T) { func TestAuthenticateAppliesTheRoutePolicyDefault(t *testing.T) { message, err := DecodeClientMessage([]byte( - `{"type":"authenticate","protocol":"piik-v23","roomId":"1234","role":"host",` + + `{"type":"authenticate","protocol":"piik-v24","roomId":"1234","role":"host",` + `"token":"` + repeat("a", 43) + `","clientId":"client_12345678"}`)) if err != nil { t.Fatal(err) @@ -721,7 +721,7 @@ func TestAuthenticateAppliesTheRoutePolicyDefault(t *testing.T) { func TestAuthenticateAcceptsPersistentRoomSessionOptIn(t *testing.T) { message, err := DecodeClientMessage([]byte( - `{"type":"authenticate","protocol":"piik-v23","roomId":"1234","role":"host",` + + `{"type":"authenticate","protocol":"piik-v24","roomId":"1234","role":"host",` + `"token":"` + repeat("a", 43) + `","clientId":"client_12345678","roomSession":true,"roomOnly":true}`)) if err != nil { t.Fatal(err) @@ -731,7 +731,7 @@ func TestAuthenticateAcceptsPersistentRoomSessionOptIn(t *testing.T) { t.Fatalf("decoded %#v, want room-session opt-in and room-only state", message) } if _, err := DecodeClientMessage([]byte( - `{"type":"authenticate","protocol":"piik-v23","roomId":"1234","role":"host",` + + `{"type":"authenticate","protocol":"piik-v24","roomId":"1234","role":"host",` + `"token":"` + repeat("a", 43) + `","clientId":"client_12345678","roomOnly":true}`)); err == nil { t.Fatal("accepted roomOnly without roomSession opt-in") } diff --git a/internal/server/protocol/types.go b/internal/server/protocol/types.go index 4bfd18683..e026f1424 100644 --- a/internal/server/protocol/types.go +++ b/internal/server/protocol/types.go @@ -37,7 +37,6 @@ var ( var ( qualityResolutions = []string{"480p", "720p", "1080p", "1440p"} degradationPreferences = []string{"maintain-resolution", "balanced", "maintain-framerate"} - screenAudioQualities = []string{"saver", "music", "very-high"} mediaRoutePhases = []string{"prepare", "active"} routeDemandReasons = []string{"join", "edge-unavailable", "parent-departed", "capacity-reduction", "sfu-bootstrap", "direct-convergence", "quality-convergence", "root-convergence"} routeDiagnosticFinalRoutes = []string{"direct", "sfu", "waiting", "failed"} @@ -112,7 +111,7 @@ type QualitySettings struct { MaxFramerate Int `json:"maxFramerate"` MaxBitrate Int `json:"maxBitrate"` DegradationPreference string `json:"degradationPreference"` - ScreenAudioQuality string `json:"screenAudioQuality,omitempty"` + MaxAudioBitrate Int `json:"maxAudioBitrate,omitempty"` } // UnmarshalJSON implements json.Unmarshaler. @@ -127,7 +126,7 @@ func (v *QualitySettings) UnmarshalJSON(data []byte) error { "resolution", "maxFramerate", "maxBitrate", "degradationPreference"); err != nil { return err } - if err := present.optional("screenAudioQuality"); err != nil { + if err := present.optional("maxAudioBitrate"); err != nil { return err } if !enumOf(v.Resolution, qualityResolutions...) { @@ -142,8 +141,8 @@ func (v *QualitySettings) UnmarshalJSON(data []byte) error { if !enumOf(v.DegradationPreference, degradationPreferences...) { return fmt.Errorf("unknown degradationPreference %q", v.DegradationPreference) } - if present.has("screenAudioQuality") && !enumOf(v.ScreenAudioQuality, screenAudioQualities...) { - return fmt.Errorf("unknown screenAudioQuality %q", v.ScreenAudioQuality) + if present.has("maxAudioBitrate") && !inRangeInt(v.MaxAudioBitrate, MinAudioBitrate, MaxAudioBitrate) { + return errors.New("maxAudioBitrate is out of range") } return nil } diff --git a/internal/server/sfu/media.go b/internal/server/sfu/media.go index 63452c0c0..ad90a3efd 100644 --- a/internal/server/sfu/media.go +++ b/internal/server/sfu/media.go @@ -4,10 +4,12 @@ import ( "context" "errors" "slices" + "strconv" "strings" "sync" "github.com/TNTcraftHIM/Piik/internal/media/forwarding" + "github.com/TNTcraftHIM/Piik/internal/server/protocol" "github.com/pion/interceptor" "github.com/pion/rtcp" "github.com/pion/webrtc/v4" @@ -342,7 +344,7 @@ func (media *Media) AcceptPublisherOffer(fence ResourceFence, connectionID strin return webrtc.SessionDescription{}, errors.New("SFU publisher codec does not match its declaration") } if transceiver.Kind() == webrtc.RTPCodecTypeAudio { - selected[0].SDPFmtpLine = "minptime=10;useinbandfec=1;stereo=1;usedtx=0" + selected[0].SDPFmtpLine = "minptime=10;useinbandfec=1;stereo=1;usedtx=0;maxaveragebitrate=" + strconv.Itoa(protocol.MaxAudioBitrate) } if err = transceiver.SetCodecPreferences(selected[:1]); err != nil { return webrtc.SessionDescription{}, err @@ -639,6 +641,11 @@ func (media *Media) ApplySubscriberAnswer(fence SubscriptionFence, connectionID if answer.Type != webrtc.SDPTypeAnswer { return errors.New("SFU subscriber requires an answer") } + if subscriber.transport.AudioSender != nil { + if err := forwarding.CheckAudioReceiveLimit(answer.SDP, protocol.MaxAudioBitrate); err != nil { + return err + } + } return subscriber.transport.PC.SetRemoteDescription(answer) } diff --git a/internal/server/sfu/media_test.go b/internal/server/sfu/media_test.go index 8720cb0d2..3fe701873 100644 --- a/internal/server/sfu/media_test.go +++ b/internal/server/sfu/media_test.go @@ -296,6 +296,18 @@ func runMediaFanout(t *testing.T, codec string, layerCount int) { } }) check(receiver.SetRemoteDescription(offer)) + for _, transceiver := range receiver.GetTransceivers() { + if transceiver.Kind() != webrtc.RTPCodecTypeAudio { + continue + } + for _, codec := range transceiver.Receiver().GetParameters().Codecs { + if codec.MimeType == webrtc.MimeTypeOpus { + codec.SDPFmtpLine = "stereo=1;maxaveragebitrate=510000" + check(transceiver.SetCodecPreferences([]webrtc.RTPCodecParameters{codec})) + break + } + } + } answer, answerErr := receiver.CreateAnswer(nil) check(answerErr) check(service.ApplySubscriberAnswer(subscription, "subscriber_123456", waitGathering(receiver, answer))) diff --git a/internal/server/signal/server_test.go b/internal/server/signal/server_test.go index 092fe005d..16ec7c23e 100644 --- a/internal/server/signal/server_test.go +++ b/internal/server/signal/server_test.go @@ -2434,9 +2434,9 @@ func TestSignalBindsNewShareQualityAndPreservesActiveShareQualityAcrossReconnect "maxFramerate": 60, "maxBitrate": 8_000_000, "degradationPreference": "maintain-resolution", - "screenAudioQuality": "very-high", + "maxAudioBitrate": 192000, } - initialJSON := `{"resolution":"1440p","maxFramerate":60,"maxBitrate":8000000,"degradationPreference":"maintain-resolution","screenAudioQuality":"very-high"}` + initialJSON := `{"resolution":"1440p","maxFramerate":60,"maxBitrate":8000000,"degradationPreference":"maintain-resolution","maxAudioBitrate":192000}` firstGeneration := "first_share_generation_12345678" viewer := openClient(t, h) authenticate(t, viewer, h.room, protocol.RoleViewer, "quality-viewer-client", 1, "", presenceOptions{}) @@ -2452,9 +2452,9 @@ func TestSignalBindsNewShareQualityAndPreservesActiveShareQualityAcrossReconnect "maxFramerate": 15, "maxBitrate": 2_000_000, "degradationPreference": "maintain-framerate", - "screenAudioQuality": "saver", + "maxAudioBitrate": 64000, } - reconnectJSON := `{"resolution":"480p","maxFramerate":15,"maxBitrate":2000000,"degradationPreference":"maintain-framerate","screenAudioQuality":"saver"}` + reconnectJSON := `{"resolution":"480p","maxFramerate":15,"maxBitrate":2000000,"degradationPreference":"maintain-framerate","maxAudioBitrate":64000}` reconnectedHost := openClient(t, h) reconnectedAuth := authenticate(t, reconnectedHost, h.room, protocol.RoleHost, "quality-host-client", 1, firstGeneration, presenceOptions{qualitySettings: reconnectSettings}) @@ -2469,9 +2469,9 @@ func TestSignalBindsNewShareQualityAndPreservesActiveShareQualityAcrossReconnect "maxFramerate": 30, "maxBitrate": 3_000_000, "degradationPreference": "balanced", - "screenAudioQuality": "music", + "maxAudioBitrate": 128000, } - updatedJSON := `{"resolution":"720p","maxFramerate":30,"maxBitrate":3000000,"degradationPreference":"balanced","screenAudioQuality":"music"}` + updatedJSON := `{"resolution":"720p","maxFramerate":30,"maxBitrate":3000000,"degradationPreference":"balanced","maxAudioBitrate":128000}` reconnectedHost.sendJSON(map[string]any{"type": "set-quality-settings", "qualitySettings": updatedSettings}) expectEqual(t, viewer.next("quality-settings").raw, `{"type":"quality-settings","qualitySettings":`+updatedJSON+`}`) diff --git a/scripts/app-media-gate.ts b/scripts/app-media-gate.ts index 27b1ad6d3..67b803892 100644 --- a/scripts/app-media-gate.ts +++ b/scripts/app-media-gate.ts @@ -355,7 +355,7 @@ async function browserMediaGate(input: { rejectRequest(new Error("Native request timed out: " + type)); }, 8_000); pending.set(id, { resolve: resolveRequest, reject: rejectRequest, timer }); - socket!.send(JSON.stringify({ version: 9, id, type, ...fields })); + socket!.send(JSON.stringify({ version: 10, id, type, ...fields })); }); }; socket.onmessage = (event) => { @@ -440,7 +440,7 @@ async function browserMediaGate(input: { maxFramerate: 30, maxBitrate: 3_000_000, degradationPreference: "balanced", - screenAudioQuality: "music", + maxAudioBitrate: 128000, }, }); if (started.shareId !== shareId || started.audio !== result.audioAvailable) { @@ -592,7 +592,7 @@ async function browserMediaGate(input: { maxFramerate: 60, maxBitrate: 12_000_000, degradationPreference: "maintain-framerate", - screenAudioQuality: "very-high", + maxAudioBitrate: 192000, }, }); const liveDeadline = performance.now() + 15_000; @@ -618,7 +618,7 @@ async function browserMediaGate(input: { maxFramerate: 15, maxBitrate: 2_000_000, degradationPreference: "maintain-resolution", - screenAudioQuality: "saver", + maxAudioBitrate: 64000, }, }); const framesBeforeResume = Math.min( diff --git a/scripts/browser-local-pool-page.js b/scripts/browser-local-pool-page.js index 31d32218a..a2ead5744 100644 --- a/scripts/browser-local-pool-page.js +++ b/scripts/browser-local-pool-page.js @@ -18,7 +18,7 @@ const quietStart = settings.get('quietStart') === '1'; const selectedProfile = () => ({ resolution: high ? '1080p' : '480p', maxFramerate: 30, maxBitrate: high ? 5000000 : 1700000, degradationPreference: settings.get('preference') || (high ? 'maintain-resolution' : 'balanced'), - screenAudioQuality: 'music' }); + maxAudioBitrate: Number(settings.get('audioBitrate') || 128000) }); const productPool = mode === 'carrier' ? new ProbePool() : null; const productPeers = [], peers = [], tracks = [], edges = [], monitors = [], audioMonitors = []; const errors = [], sourcePulses = [], controls = [], visibility = [], productTrace = [], sourceFrames = []; @@ -193,7 +193,8 @@ async function statistics() { const result = {}; for (const edge of edges) { const metrics = edge.peer.getSnapshot().metrics; - result[edge.role] = { send: selectReport(await edge.send.getStats()), + result[edge.role] = { audioParameters: edge.peer.getSnapshot().audioSenderParameters, + send: selectReport(await edge.send.getStats()), receive: selectReport(await edge.receive.getStats()), details: Object.fromEntries(['frameWidth', 'frameHeight', 'framesPerSecond', 'bitrateKbps', 'qualityLimitationReason', 'nativeEdgeQualityState', 'intervalFramesSent', 'intervalFramesEncoded'] diff --git a/scripts/browser-local-pool-probe.ts b/scripts/browser-local-pool-probe.ts index 77afc1358..132a6b298 100644 --- a/scripts/browser-local-pool-probe.ts +++ b/scripts/browser-local-pool-probe.ts @@ -11,7 +11,7 @@ const mode = process.argv[2] || 'carrier', args = process.argv.slice(3); const flags = new Set(['--product', '--1080', '--h264', '--single', '--av', '--late', '--background', '--native-source', '--lifecycle', '--network', '--auto', '--relay', '--debug', '--short-pulse', '--display', '--quiet-start']); if (!['ordinary', 'carrier'].includes(mode)) throw Error('Expected ordinary|carrier'); -for (const arg of args) if (!flags.has(arg) && !arg.startsWith('--rate=') && !arg.startsWith('--preference=')) +for (const arg of args) if (!flags.has(arg) && !arg.startsWith('--rate=') && !arg.startsWith('--preference=') && !arg.startsWith('--audio-bitrate=')) throw Error('Unsupported option: ' + arg); const codec = args.includes('--h264') ? 'H264' : 'VP8', network = args.includes('--network'); const automatic = args.includes('--auto'), high = args.includes('--1080'), av = args.includes('--av'); @@ -22,6 +22,8 @@ const nativeSource = background || args.includes('--native-source'), lifecycle = const relay = args.includes('--relay'); const shortPulse = args.includes('--short-pulse'); const rate = Number(args.find(arg => arg.startsWith('--rate='))?.split('=')[1] || 120000); +const audioBitrate = Number(args.find(arg => arg.startsWith('--audio-bitrate='))?.split('=')[1] || 128000); +if (!Number.isInteger(audioBitrate) || audioBitrate < 6000 || audioBitrate > 320000) throw Error('Audio bitrate must be 6000..320000 bps'); const preference = args.find(arg => arg.startsWith('--preference='))?.split('=')[1] || (high ? 'maintain-resolution' : 'balanced'); if (!['balanced', 'maintain-resolution', 'maintain-framerate'].includes(preference)) throw Error('Unsupported degradation preference'); if (!Number.isInteger(rate) || rate <= 0) throw Error('Rate must be a positive integer in bps'); @@ -85,7 +87,7 @@ try { const version = await waitForVersion(debugPort, chrome); cdp = await CdpConnection.connect(version.webSocketDebuggerUrl, Date.now() + 5000); page = await createPage(cdp, 'about:blank'); - const query = new URLSearchParams({ mode, codec, rate: String(rate), preference }); + const query = new URLSearchParams({ mode, codec, rate: String(rate), preference, audioBitrate: String(audioBitrate) }); if (process.argv.includes('--debug')) query.set('debug', '1'); for (const [key, value] of Object.entries({ network, automatic, high, av, single, late, background, nativeSource, displaySource, quietStart, lifecycle, relay, shortPulse })) query.set(key, value ? '1' : '0'); diff --git a/scripts/embedded-sfu-gate.ts b/scripts/embedded-sfu-gate.ts index 38d1bb78c..a6d0fbc78 100644 --- a/scripts/embedded-sfu-gate.ts +++ b/scripts/embedded-sfu-gate.ts @@ -99,6 +99,7 @@ async function main(): Promise { high: null as { frames: number; width: number; height: number } | null, low: null as { frames: number; width: number; height: number } | null, audioEnergy: { high: 0, low: 0, recovered: 0 }, + audioUpdates: [] as Array<{ bitrate: number; frames: number; connectionRetained: boolean; energy: number }>, audioDiagnostics: null as unknown, secondRoom: twoRooms ? { connected: false, framesAfterFirstStopped: 0, audioBefore: 0, audioAfter: 0, stopped: false } : null, host: null as Snapshot | null, viewer: null as Snapshot | null, @@ -279,6 +280,19 @@ async function main(): Promise { (energy) => { result.audioEnergy.high = energy; return energy > 0; }, 5_000); if (!nativeArm) result.stereoSeparationDb = await call(viewer, "gate.audioSeparation()", 5_000); result.host = await call(host, "gate.snapshot()"); + result.stage = "numeric-audio-update"; + const audioConnection = (await call(viewer, "gate.snapshot()")).connectionId; + let previousEnergy = result.audioEnergy.high; + for (const bitrate of [320_000, 96_000]) { + if (!await call(host, `gate.setAudioBitrate(${bitrate})`)) throw new Error("Numeric audio update failed"); + const frames = await call<{ frames: number }>(viewer, "gate.waitForFrames(1280, 720)"); + const after = await call(viewer, "gate.snapshot()"); + const energy = await call(viewer, "gate.audioEnergy()", 3_000); + const retained = after.connectionId === audioConnection && after.committed; + result.audioUpdates.push({ bitrate, frames: frames.frames, connectionRetained: retained, energy }); + if (!retained || energy <= previousEnergy) throw new Error("Audio update interrupted the current media connection"); + previousEnergy = energy; + } if (displayArm) { result.stage = "display-background"; if (result.host!.capture?.displaySurface !== "browser" || result.host!.visibility !== "hidden") { diff --git a/scripts/embedded-sfu-page.ts b/scripts/embedded-sfu-page.ts index 43f3a2060..dba5c67c7 100644 --- a/scripts/embedded-sfu-page.ts +++ b/scripts/embedded-sfu-page.ts @@ -19,7 +19,7 @@ import { ViewerSfuRoute } from "../src/client/media/viewer-sfu-route"; const high: QualitySettings = { resolution: "720p", maxFramerate: 15, maxBitrate: 2_000_000, - degradationPreference: "maintain-resolution", screenAudioQuality: "saver", + degradationPreference: "maintain-resolution", maxAudioBitrate: 64000, }; const low: QualitySettings = { ...high, resolution: "480p" }; const shareGeneration = crypto.randomUUID(); @@ -316,6 +316,7 @@ export async function reconnectViewer(room: CreateRoomResponse, discardMedia = f export function snapshot() { return { authenticated, published, simulcast, decoded, committed, peerFailures, + maxAudioBitrate: profile.maxAudioBitrate, visibility: document.visibilityState, capture: source?.getVideoTracks()[0]?.getSettings(), routeEvents, routeStatus, connectionId: configuration?.connectionId, @@ -340,6 +341,15 @@ export async function audioEnergy(): Promise { return await probe(); } +export async function setAudioBitrate(maxAudioBitrate: number): Promise { + const next = { ...profile, maxAudioBitrate }; + if (native) await native.updateShare(shareGeneration, next); + if (!await hostRoute!.updateProfile(next)) return false; + profile = next; + send({ type: "set-quality-settings", qualitySettings: next }); + return true; +} + export async function waitForFrames(width: number, height: number, minimumFrames = 15) { if (!video) throw new Error("Viewer video is unavailable"); const target = video; diff --git a/src/client/locales/en.ts b/src/client/locales/en.ts index 47d334671..301449aba 100644 --- a/src/client/locales/en.ts +++ b/src/client/locales/en.ts @@ -278,7 +278,7 @@ export const en: Record = { "host.terminated.signal": "Server connection terminated; refresh to continue", "host.notice.qualityApplied": "Share settings applied", "host.notice.qualitySet": "Quality switched to {label}", - "host.notice.audioSet": "Audio quality switched to {label}", + "host.notice.audioSet": "Audio bitrate limit set to {kbps} kbps", "host.notice.partialApply": "Settings updated, but some viewer connections could not apply them", "host.notice.sourceSwitch.ok": "Share source switched", "host.notice.sourceSwitch.partial": "Source switched, but {warning}", @@ -358,6 +358,8 @@ export const en: Record = { "host.advanced.preference.balancedHint": "Weigh automatically", "host.advanced.preference.framerateHint": "Prefer frame rate", "host.advanced.audio": "Audio quality", + "host.advanced.audioBitrate": "Audio bitrate limit", + "host.audio.bitrateHint": "Audio bitrate limit: {min}–{max} kbps.", "host.advanced.audio.saver": "Standard", "host.advanced.audio.music": "Music", "host.advanced.audio.veryHigh": "Studio", diff --git a/src/client/locales/zh.ts b/src/client/locales/zh.ts index 2032e86f6..d3cbcc53c 100644 --- a/src/client/locales/zh.ts +++ b/src/client/locales/zh.ts @@ -278,7 +278,7 @@ export const zh = { "host.terminated.signal": "服务器连接已终止,请刷新后重试", "host.notice.qualityApplied": "分享设置已应用", "host.notice.qualitySet": "画质已切换为 {label}", - "host.notice.audioSet": "音频质量已切换为 {label}", + "host.notice.audioSet": "音频码率上限已设为 {kbps} kbps", "host.notice.partialApply": "分享设置已更新,但部分观看连接未能应用新参数", "host.notice.sourceSwitch.ok": "分享来源已切换", "host.notice.sourceSwitch.partial": "分享来源已切换,但{warning}", @@ -358,6 +358,8 @@ export const zh = { "host.advanced.preference.balancedHint": "自动权衡", "host.advanced.preference.framerateHint": "优先帧率", "host.advanced.audio": "音频质量", + "host.advanced.audioBitrate": "音频码率上限", + "host.audio.bitrateHint": "音频码率上限:{min}–{max} kbps。", "host.advanced.audio.saver": "普通", "host.advanced.audio.music": "音乐", "host.advanced.audio.veryHigh": "保真", diff --git a/src/client/media/quality.ts b/src/client/media/quality.ts index bf98bf565..584a12f4e 100644 --- a/src/client/media/quality.ts +++ b/src/client/media/quality.ts @@ -1,10 +1,11 @@ import { DEFAULT_QUALITY_SETTINGS, + MIN_AUDIO_BITRATE, + MAX_AUDIO_BITRATE, type DegradationPreference, type QualityProfileId, type QualityResolution, type QualitySettings, - type ScreenAudioQuality, } from "../../shared/protocol"; import { say, type CopyKey } from "../ui/copy"; import type { MediaFailure } from "../ui/media-failure"; @@ -18,7 +19,6 @@ export type { QualityProfileId, QualityResolution, QualitySettings, - ScreenAudioQuality, } from "../../shared/protocol"; export type QualityProfile = QualitySettings; @@ -30,7 +30,7 @@ export const QUALITY_PROFILES = { maxFramerate: 30, maxBitrate: 3_000_000, degradationPreference: "balanced", - screenAudioQuality: "music", + maxAudioBitrate: 128000, }, "1080p30": DEFAULT_QUALITY_SETTINGS, "1080p60": { @@ -38,7 +38,7 @@ export const QUALITY_PROFILES = { maxFramerate: 60, maxBitrate: 8_000_000, degradationPreference: "balanced", - screenAudioQuality: "music", + maxAudioBitrate: 128000, }, } as const satisfies Record; @@ -64,14 +64,15 @@ export const DEGRADATION_PREFERENCE_KEYS = { "maintain-framerate": "host.advanced.preference.framerate", } as const satisfies Record; -export const DEFAULT_SCREEN_AUDIO_QUALITY: ScreenAudioQuality = "music"; export const SCREEN_AUDIO_BITRATES = { saver: 64_000, music: 128_000, "very-high": 192_000, -} as const satisfies Record; -export const SCREEN_AUDIO_RECEIVE_MAX_BITRATE = - SCREEN_AUDIO_BITRATES["very-high"]; +} as const; +export type ScreenAudioPreset = keyof typeof SCREEN_AUDIO_BITRATES; +// Current 20 ms output leaves room for RTP/SRTP overhead within a 1200-byte packet. +export const MAX_SCREEN_AUDIO_BITRATE = 320_000; +export const SCREEN_AUDIO_RECEIVE_MAX_BITRATE = MAX_AUDIO_BITRATE; export function startupVideoProfile( profile: QualityProfile, @@ -105,16 +106,10 @@ export interface AudioSenderParameterReadback { mismatch: boolean; } -export function resolveScreenAudioQuality( - quality: ScreenAudioQuality | undefined, -): ScreenAudioQuality { - return quality ?? DEFAULT_SCREEN_AUDIO_QUALITY; -} - export function screenAudioBitrate( - quality: ScreenAudioQuality | undefined, + bitrate: number | undefined, ): number { - return SCREEN_AUDIO_BITRATES[resolveScreenAudioQuality(quality)]; + return bitrate ?? DEFAULT_QUALITY_SETTINGS.maxAudioBitrate; } export function qualitySettingsEqual( @@ -123,7 +118,7 @@ export function qualitySettingsEqual( ): boolean { return ( videoQualitySettingsEqual(left, right) && - screenAudioQualityEqual(left, right) + audioQualitySettingsEqual(left, right) ); } @@ -139,13 +134,13 @@ export function videoQualitySettingsEqual( ); } -export function screenAudioQualityEqual( +export function audioQualitySettingsEqual( left: QualitySettings, right: QualitySettings, ): boolean { return ( - resolveScreenAudioQuality(left.screenAudioQuality) === - resolveScreenAudioQuality(right.screenAudioQuality) + screenAudioBitrate(left.maxAudioBitrate) === + screenAudioBitrate(right.maxAudioBitrate) ); } @@ -398,6 +393,36 @@ export async function configureVideoSender( return senderParameterReadback(requested, applied); } +export function screenAudioSenderCeiling(current: RTCRtpSendParameters, requestedMaxBitrate: number): number { + // Some Browsers accept setParameters above the negotiated Opus receive limit. + // Keep that limit on this sender, never on the shared capture or sibling edges. + let ceiling = Math.min(requestedMaxBitrate, MAX_SCREEN_AUDIO_BITRATE); + for (const codec of current.codecs ?? []) { + if (codec.mimeType.toLowerCase() !== "audio/opus") continue; + let receiveLimit: number | undefined; + let playbackRate = 48_000; + let channels = 1; + for (const parameter of (codec.sdpFmtpLine ?? "").split(";")) { + const [name, value] = parameter.trim().split("="); + const key = name?.trim().toLowerCase(); + const number = Number(value); + if (key === "stereo") channels = value?.trim() === "1" ? 2 : 1; + if (key === "maxplaybackrate" && Number.isInteger(number) && number >= 8_000 && number <= 48_000) { + playbackRate = number; + } + // RFC 7587 ignores out-of-range receive preferences. + if (key === "maxaveragebitrate" && Number.isInteger(number) && number >= MIN_AUDIO_BITRATE && number <= MAX_AUDIO_BITRATE) { + receiveLimit = Math.min(receiveLimit ?? MAX_AUDIO_BITRATE, number); + } + } + // Match libwebrtc's default when the peer omits a valid explicit ceiling; + // omission must not let setParameters override the default with a higher rate. + const defaultBitrate = (playbackRate <= 8_000 ? 12_000 : playbackRate <= 16_000 ? 20_000 : 32_000) * channels; + ceiling = Math.min(ceiling, receiveLimit ?? defaultBitrate); + } + return ceiling; +} + export async function configureScreenAudioSender( sender: RTCRtpSender, requestedMaxBitrate: number, @@ -410,7 +435,7 @@ export async function configureScreenAudioSender( if (parameters.encodings.length === 0) { parameters.encodings = [{}]; } - parameters.encodings[0]!.maxBitrate = requestedMaxBitrate; + parameters.encodings[0]!.maxBitrate = screenAudioSenderCeiling(current, requestedMaxBitrate); await sender.setParameters(parameters); const appliedMaxBitrate = sender.getParameters().encodings[0]?.maxBitrate ?? null; diff --git a/src/client/native/wire.ts b/src/client/native/wire.ts index 49db1be37..05b3e39f6 100644 --- a/src/client/native/wire.ts +++ b/src/client/native/wire.ts @@ -1,10 +1,10 @@ import { z } from "zod"; -import { sfuMediaSchema } from "../../shared/protocol"; +import { sfuMediaSchema, maxAudioBitrateSchema } from "../../shared/protocol"; -export const NATIVE_CLIENT_PROTOCOL = 9; +export const NATIVE_CLIENT_PROTOCOL = 10; export const NATIVE_CLIENT_PORT_START = 39_721; export const NATIVE_CLIENT_PORT_END = 39_730; -export const NATIVE_CLIENT_SUBPROTOCOL = "piik-client-v9"; +export const NATIVE_CLIENT_SUBPROTOCOL = "piik-client-v10"; const decimalIdentifierSchema = z.string().regex(/^[1-9]\d{0,19}$/); const opaqueIdentifierSchema = z @@ -33,6 +33,7 @@ export const nativeHealthSchema = nativeDiscoveryIdentitySchema.extend({ .regex(/^[A-Za-z0-9_-]+$/), nativeMedia: z .object({ + maxAudioBitrate: maxAudioBitrateSchema.optional(), receiverReuse: z.boolean().default(false), video: z.boolean().default(false), processAudio: z.boolean().default(false), diff --git a/src/client/pages/HostPage.tsx b/src/client/pages/HostPage.tsx index 2ba9e932e..5ed8d233e 100644 --- a/src/client/pages/HostPage.tsx +++ b/src/client/pages/HostPage.tsx @@ -10,6 +10,7 @@ import { import { DEFAULT_ROUTE_POLICY, MAX_VIEWER_PASSWORD_LENGTH, + MIN_AUDIO_BITRATE, viewerPasswordSchema, type CreateRoomResponse, type ClientMessage, @@ -130,14 +131,15 @@ import { QUALITY_RESOLUTIONS, qualitySettingsEqual, qualitySettingsLabel, - resolveScreenAudioQuality, + screenAudioBitrate, + MAX_SCREEN_AUDIO_BITRATE, SCREEN_AUDIO_BITRATES, setMediaPaused, videoQualitySettingsEqual, type DegradationPreference, type QualityResolution, type QualitySettings, - type ScreenAudioQuality, + type ScreenAudioPreset, } from "../media/quality"; import { HostSfuRoute } from "../media/host-sfu-route"; import { BrowserEncodingPool } from "../media/browser-encoding-pool"; @@ -233,7 +235,7 @@ const PREFERENCE_PRESENTATION: Record< }, }; -const AUDIO_QUALITY_CAPTIONS: Record = { +const AUDIO_QUALITY_CAPTIONS: Record = { saver: "host.advanced.audio.saver", music: "host.advanced.audio.music", "very-high": "host.advanced.audio.veryHigh", @@ -1644,13 +1646,13 @@ export function HostPage({ } } - function changeScreenAudioQuality( - screenAudioQuality: ScreenAudioQuality, + function changeMaxAudioBitrate( + maxAudioBitrate: number, ): void { if (phase === "starting") { return; } - const next = { ...advancedQualityRef.current, screenAudioQuality }; + const next = { ...advancedQualityRef.current, maxAudioBitrate }; void changeQuality(next); } @@ -1730,8 +1732,8 @@ export function HostPage({ previousProfile.resolution !== nextProfile.resolution || previousProfile.maxFramerate !== nextProfile.maxFramerate; const audioChanged = - resolveScreenAudioQuality(previousProfile.screenAudioQuality) !== - resolveScreenAudioQuality(nextProfile.screenAudioQuality); + screenAudioBitrate(previousProfile.maxAudioBitrate) !== + screenAudioBitrate(nextProfile.maxAudioBitrate); try { const nativeClient = nativeClientRef.current; const nativeShareGeneration = nativeShareGenerationRef.current; @@ -1822,7 +1824,7 @@ export function HostPage({ : null; const successNotice = audioChanged && !videoChanged - ? say("host.notice.audioSet", { label: say(AUDIO_QUALITY_CAPTIONS[resolveScreenAudioQuality(nextProfile.screenAudioQuality)]) }) + ? say("host.notice.audioSet", { kbps: String(screenAudioBitrate(nextProfile.maxAudioBitrate) / 1_000) }) : videoChanged && audioChanged ? say("host.notice.qualityApplied") : say("host.notice.qualitySet", { @@ -3592,6 +3594,9 @@ export function HostPage({ const titleContent = titleFrames(hostStatus.titleFrameKey); useDocumentTitle([room?.roomId, titleContent.label, hostStatus.titleMarker], titleContent.variations, `${lang}:${vis}:${hostStatus.titleFrameKey}:${room?.roomId ?? ""}`); + const audioBitrateCeiling = Math.min(MAX_SCREEN_AUDIO_BITRATE, nativeActive + ? nativeClientRef.current?.health.nativeMedia.maxAudioBitrate ?? MAX_SCREEN_AUDIO_BITRATE + : MAX_SCREEN_AUDIO_BITRATE); return (
@@ -3828,7 +3833,7 @@ export function HostPage({ void changeQuality({ ...QUALITY_PROFILES[id], - screenAudioQuality: resolveScreenAudioQuality(advancedQualityRef.current.screenAudioQuality), + maxAudioBitrate: screenAudioBitrate(advancedQualityRef.current.maxAudioBitrate), })} /> } picture={<> @@ -4016,16 +4021,17 @@ export function HostPage({ {( Object.keys( AUDIO_QUALITY_CAPTIONS, - ) as ScreenAudioQuality[] + ) as ScreenAudioPreset[] ).map((audioQuality) => ( audioBitrateCeiling} title={t("host.audio.title", { label: t(AUDIO_QUALITY_CAPTIONS[audioQuality]), kbps: String( @@ -4033,7 +4039,7 @@ export function HostPage({ ), })} hint="hint-audio-quality" - onClick={() => changeScreenAudioQuality(audioQuality)} + onClick={() => changeMaxAudioBitrate(SCREEN_AUDIO_BITRATES[audioQuality])} > @@ -4042,6 +4048,36 @@ export function HostPage({ ))}
+ + + } technical={<> diff --git a/src/client/sfu/publisher.ts b/src/client/sfu/publisher.ts index 8f79aa124..46514e5f5 100644 --- a/src/client/sfu/publisher.ts +++ b/src/client/sfu/publisher.ts @@ -10,6 +10,7 @@ import { needsStartupVideoProfile, QUALITY_RESOLUTIONS, screenAudioBitrate, + screenAudioSenderCeiling, senderParameterWarning, STARTUP_VIDEO_ENCODED_FRAMES, startupVideoProfile, @@ -108,6 +109,16 @@ export class SfuPublisher { if (description.type !== "answer") throw new Error("SFU publisher expected an answer"); await peer.waitForOperation(() => peer.pc.setRemoteDescription(description)); + const audio = this.audioSender; + if (this.profile && audio?.track && screenAudioSenderCeiling(audio.getParameters(), + screenAudioBitrate(this.profile.maxAudioBitrate)) !== audio.getParameters().encodings[0]?.maxBitrate) { + void this.enqueue(async () => { + if (this.peer !== peer || !this.profile) return false; + const applied = await this.configure(this.profile, false); + if (!applied && this.peer === peer) this.fail("transport"); + return applied; + }).catch(() => this.fail("transport")); + } }, async (activeCount) => { await this.enqueue(async () => { @@ -181,7 +192,7 @@ export class SfuPublisher { direction: "sendonly", streams: [stream], sendEncodings: [ - { maxBitrate: screenAudioBitrate(profile.screenAudioQuality) }, + { maxBitrate: screenAudioBitrate(profile.maxAudioBitrate) }, ], }); const opus = RTCRtpSender.getCapabilities("audio")?.codecs.filter( @@ -475,7 +486,7 @@ export class SfuPublisher { bitrate: encoding.maxBitrate!, })), audio: this.audio !== null, - audioBitrate: screenAudioBitrate(profile.screenAudioQuality), + audioBitrate: screenAudioBitrate(profile.maxAudioBitrate), }; } @@ -510,7 +521,7 @@ export class SfuPublisher { try { const audio = await peer.waitForOperation(() => configureScreenAudioSender( audioSender, - screenAudioBitrate(profile.screenAudioQuality), + screenAudioBitrate(profile.maxAudioBitrate), )); if (this.peer !== peer) return false; this.audioWarning = audioSenderParameterWarning(audio); diff --git a/src/client/styles.css b/src/client/styles.css index d167f5691..e93f3ae56 100644 --- a/src/client/styles.css +++ b/src/client/styles.css @@ -1176,6 +1176,8 @@ html[data-theme="dark"] .lr-invite-url { background: #1a2634; border-color: #334 .lr-input:focus-within { border-color: var(--action); outline: 3px solid var(--action); outline-offset: 2px; } .lr-input input { min-width: 0; flex: 1; height: 100%; border: 0; outline: 0; background: transparent; font: inherit; font-size: 16px; } .lr-input input[type="password"]:not(:placeholder-shown) { letter-spacing: .25em; } +.lr-input.lr-audio-bitrate { width: 160px; max-width: 100%; font-family: var(--mono); } +.lr-audio-bitrate span { color: var(--muted); font-size: 12px; } .lr-welcome { width: 100%; min-width: 0; max-width: 100%; margin: 0; overflow: hidden; color: inherit; font-size: 15px; font-weight: 600; line-height: 1.7; text-align: center; white-space: nowrap; text-overflow: ellipsis; } .lr-welcome:not(.is-cipher)::before { content: "✦"; color: #d7982b; margin-right: 8px; } .lr-welcome-cipher { display: inline-flex; align-items: center; gap: 8px; vertical-align: middle; opacity: .72; white-space: nowrap; } diff --git a/src/client/webrtc/host-peer.ts b/src/client/webrtc/host-peer.ts index ad8fbecbb..b4322ea36 100644 --- a/src/client/webrtc/host-peer.ts +++ b/src/client/webrtc/host-peer.ts @@ -11,15 +11,14 @@ import { configureScreenAudioSender, configureVideoSender, needsStartupVideoProfile, - resolveScreenAudioQuality, screenAudioBitrate, - screenAudioQualityEqual, + screenAudioSenderCeiling, + audioQualitySettingsEqual, STARTUP_VIDEO_ENCODED_FRAMES, startupVideoProfile, videoQualitySettingsEqual, type AudioSenderParameterReadback, type QualityProfile, - type ScreenAudioQuality, } from "../media/quality"; import { cloneSenderVideoTrack, stopSenderVideoTrack } from "../media/sender-video-track"; import { EMPTY_METRICS, type PeerSnapshot } from "../types"; @@ -93,7 +92,7 @@ export class HostPeer { private statsInFlight = false; private statsSamplingBlocked = false; private appliedVideoProfile: QualityProfile | null = null; - private appliedAudioQuality: ScreenAudioQuality | null = null; + private appliedAudioBitrate: number | null = null; private appliedAudioSenderParameters: | AudioSenderParameterReadback | null = null; @@ -369,7 +368,7 @@ export class HostPeer { : profile; const changed = !videoQualitySettingsEqual(this.desiredProfile, profile) || - !screenAudioQualityEqual(this.desiredProfile, profile); + !audioQualitySettingsEqual(this.desiredProfile, profile); this.desiredProfile = profile; // Connection recovery replays the same intent, not a new user mutation. const requestedRevision = changed ? ++this.profileRevision : this.profileRevision; @@ -409,8 +408,8 @@ export class HostPeer { effectiveVideoProfile, )); const updateAudio = - this.appliedAudioQuality !== - resolveScreenAudioQuality(profile.screenAudioQuality); + this.appliedAudioBitrate !== + screenAudioBitrate(profile.maxAudioBitrate); if (!updateVideo && !updateAudio) { return true; } @@ -553,7 +552,7 @@ export class HostPeer { this.awaitingReconnect = false; if (replayProfile) { this.appliedVideoProfile = null; - this.appliedAudioQuality = null; + this.appliedAudioBitrate = null; this.appliedAudioSenderParameters = null; } if (replayProfile || !this.startupVideoProfilePending) { @@ -746,6 +745,20 @@ export class HostPeer { if (!this.ownsAnswer(epoch)) { return; } + const audio = this.audioSender; + if (audio?.track && screenAudioSenderCeiling(audio.getParameters(), + screenAudioBitrate(this.desiredProfile.maxAudioBitrate)) !== audio.getParameters().encodings[0]?.maxBitrate) { + // Never await sender work inside negotiation: source replacement can + // already be waiting for this negotiation queue to finish. + void this.enqueueSenderMutation(async () => { + if (this.disposed || !this.videoSender || this.audioSender !== audio) return; + const revision = this.profileRevision; + const applied = await this.configureSender(this.videoSender, audio, { + profile: this.desiredProfile, profileRevision: revision, video: false, audio: true, + }); + if (!applied && !this.disposed && revision === this.profileRevision) this.fail(); + }).catch(error => this.fail(error)); + } await this.flushCandidates(); if (this.ownsAnswer(epoch)) { this.clearAnswerTimer(); @@ -925,7 +938,7 @@ export class HostPeer { try { audioSenderParameters = await this.waitForOperation(() => configureScreenAudioSender( audioSender, - screenAudioBitrate(profile.screenAudioQuality), + screenAudioBitrate(profile.maxAudioBitrate), )); } catch (error) { if (this.disposed) return false; @@ -951,8 +964,8 @@ export class HostPeer { this.appliedVideoProfile = profile; } if (mutation.audio && audioSucceeded) { - this.appliedAudioQuality = resolveScreenAudioQuality( - profile.screenAudioQuality, + this.appliedAudioBitrate = screenAudioBitrate( + profile.maxAudioBitrate, ); this.appliedAudioSenderParameters = audioSenderParameters; } diff --git a/src/shared/protocol.ts b/src/shared/protocol.ts index 9a6b2cc7e..87ec7301a 100644 --- a/src/shared/protocol.ts +++ b/src/shared/protocol.ts @@ -8,7 +8,7 @@ import { NAT_TRAVERSAL_PATHS } from "./nat-candidate.js"; export const MAX_VIEWERS_PER_ROOM_LIMIT = 20; export const MAX_PARTICIPANTS_PER_ROOM_LIMIT = MAX_VIEWERS_PER_ROOM_LIMIT + 1; export const MAX_SIGNAL_BYTES = 64 * 1024; -export const SIGNALING_PROTOCOL = "piik-v23"; +export const SIGNALING_PROTOCOL = "piik-v24"; export const SIGNAL_CLOSE_CODES = { serviceRestart: 1012, sessionReplaced: 4001, @@ -197,12 +197,9 @@ export type DegradationPreference = z.infer< typeof degradationPreferenceSchema >; -export const screenAudioQualitySchema = z.enum([ - "saver", - "music", - "very-high", -]); -export type ScreenAudioQuality = z.infer; +export const MIN_AUDIO_BITRATE = 6_000; +export const MAX_AUDIO_BITRATE = 510_000; +export const maxAudioBitrateSchema = z.number().int().min(MIN_AUDIO_BITRATE).max(MAX_AUDIO_BITRATE); export const qualitySettingsSchema = z .object({ @@ -210,7 +207,7 @@ export const qualitySettingsSchema = z maxFramerate: z.number().int().min(15).max(60), maxBitrate: z.number().int().min(2_000_000).max(12_000_000), degradationPreference: degradationPreferenceSchema, - screenAudioQuality: screenAudioQualitySchema.optional(), + maxAudioBitrate: maxAudioBitrateSchema.optional(), }) .strict(); export type QualitySettings = z.infer; @@ -219,7 +216,7 @@ export const DEFAULT_QUALITY_SETTINGS = { maxFramerate: 30, maxBitrate: 5_000_000, degradationPreference: "balanced", - screenAudioQuality: "music", + maxAudioBitrate: 128000, } as const satisfies QualitySettings; export const routePolicySchema = z diff --git a/tests/browser-debug.test.ts b/tests/browser-debug.test.ts index 50ab47648..2001fe65e 100644 --- a/tests/browser-debug.test.ts +++ b/tests/browser-debug.test.ts @@ -72,7 +72,7 @@ describe("opt-in Browser diagnostics", () => { data: "secret-pixels", payload: "secret-audio", roomInteraction: { payload: { kind: "chat", text: "private-chat-contents" } }, viewerGrant: "secret-grant", passwordHash: "secret-hash", - transportError: "piik-client-v9.secret-capability Basic secret-basic\nCookie: session=secret-session; refresh=secret-refresh", + transportError: "piik-client-v10.secret-capability Basic secret-basic\nCookie: session=secret-session; refresh=secret-refresh", }); const report = await page.__PIIK_DEBUG__!.export(); for (const secret of ["secret-password", "secret-auth", "secret-query", "secret-fragment", "secret-bearer", @@ -192,13 +192,13 @@ describe("opt-in Browser diagnostics", () => { const token = "x".repeat(43); class Socket extends EventTarget { static OPEN = 1; static CLOSING = 2; - readyState = 1; protocol = `piik-client-v9.${token}`; + readyState = 1; protocol = `piik-client-v10.${token}`; constructor() { super(); queueMicrotask(() => this.dispatchEvent(new Event("open"))); } close() { this.readyState = 3; } send(raw: string) { const request = JSON.parse(raw); queueMicrotask(() => this.dispatchEvent(Object.assign(new Event("message"), { - data: JSON.stringify({ version: 9, id: request.id, + data: JSON.stringify({ version: 10, id: request.id, type: request.type === "hello" ? "ready" : "share-updated", ...(request.shareId ? { shareId: request.shareId } : {}) }), }))); @@ -206,7 +206,7 @@ describe("opt-in Browser diagnostics", () => { } vi.stubGlobal("WebSocket", Socket); vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ - protocol: 9, service: "piik-client", port: 39721, instanceToken: token, + protocol: 10, service: "piik-client", port: 39721, instanceToken: token, nativeMedia: { video: true, processAudio: false, systemAudio: true, hardwareH264: true, softwareVP8: true }, }), { status: 200 }))); const debug = await import("../src/client/lib/debug"); @@ -236,7 +236,7 @@ describe("opt-in Browser diagnostics", () => { const { NativeClient } = await import("../src/client/native/client"); await expect(NativeClient.connect()).rejects.toMatchObject({ name: "NativeCompatibilityError" }); expect(page.__PIIK_DEBUG__!.events()).toContainEqual(expect.objectContaining({ - scope: "native", event: "incompatible", details: { actualProtocol: 8, expectedProtocol: 9 }, + scope: "native", event: "incompatible", details: { actualProtocol: 8, expectedProtocol: 10 }, })); expect(await debug.exportBrowserDebug()).not.toContain(token); }); diff --git a/tests/client-state.test.ts b/tests/client-state.test.ts index 3cb65de19..368ed2787 100644 --- a/tests/client-state.test.ts +++ b/tests/client-state.test.ts @@ -1728,7 +1728,7 @@ describe("client signaling recovery policy", () => { maxFramerate: 30, maxBitrate: 5_000_000, degradationPreference: "balanced", - screenAudioQuality: "music", + maxAudioBitrate: 128000, } as const; const firstPending = { ...initialQuality, @@ -2045,7 +2045,7 @@ describe("client signaling recovery policy", () => { maxFramerate: 30, maxBitrate: 5_000_000, degradationPreference: "balanced", - screenAudioQuality: "music", + maxAudioBitrate: 128000, }, routePolicy: DEFAULT_ROUTE_POLICY, }); diff --git a/tests/fixtures/wire-samples.json b/tests/fixtures/wire-samples.json index 2d929e46d..e08c3df43 100644 --- a/tests/fixtures/wire-samples.json +++ b/tests/fixtures/wire-samples.json @@ -1,6 +1,6 @@ { "constants": { - "signalingProtocol": "piik-v23", + "signalingProtocol": "piik-v24", "maxViewersPerRoomLimit": 20, "maxParticipantsPerRoomLimit": 21, "maxSignalBytes": 65536, @@ -29,7 +29,7 @@ "maxFramerate": 30, "maxBitrate": 5000000, "degradationPreference": "balanced", - "screenAudioQuality": "music" + "maxAudioBitrate": 128000 }, "defaultRoutePolicy": { "peerOnly": false, @@ -49,79 +49,79 @@ { "name": "authenticate-host-minimal", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\"}", "valid": true }, { "name": "authenticate-host-full", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"shareGeneration\":\"share_generation_12345678\",\"sharingPaused\":false,\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"viewerPresence\":true,\"displayName\":\"小明\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"shareGeneration\":\"share_generation_12345678\",\"sharingPaused\":false,\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"viewerPresence\":true,\"displayName\":\"小明\"}", "valid": true }, { "name": "authenticate-host-carries-viewer-grant", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbbg\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbbg\"}", "valid": false }, { "name": "authenticate-host-carries-viewer-password", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"viewerPassword\":\"easy-password\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"viewerPassword\":\"easy-password\"}", "valid": false }, { "name": "authenticate-host-short-token", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"short\",\"clientId\":\"client_12345678\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"short\",\"clientId\":\"client_12345678\"}", "valid": false }, { "name": "authenticate-host-unknown-key", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"admin\":true}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"admin\":true}", "valid": false }, { "name": "authenticate-viewer-minimal", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", "valid": true }, { "name": "authenticate-viewer-full", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbbg\",\"viewerPassword\":\"easy-password\",\"displayName\":\"Café 朋友\",\"viewerPresence\":true}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbbg\",\"viewerPassword\":\"easy-password\",\"displayName\":\"Café 朋友\",\"viewerPresence\":true}", "valid": true }, { "name": "authenticate-viewer-carries-host-token", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"}", "valid": false }, { "name": "authenticate-viewer-carries-route-policy", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false}}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false}}", "valid": false }, { "name": "authenticate-viewer-malformed-grant", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbb\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbb\"}", "valid": false }, { "name": "authenticate-viewer-grant-bad-final-character", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbbb\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbbb\"}", "valid": false }, { "name": "authenticate-viewer-unknown-capabilities", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"capabilities\":{\"peerIceTurn\":true}}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"capabilities\":{\"peerIceTurn\":true}}", "valid": false }, { @@ -139,55 +139,55 @@ { "name": "authenticate-viewer-leading-zero-room", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"0123\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"0123\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", "valid": false }, { "name": "authenticate-viewer-long-room", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"11111\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"11111\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", "valid": false }, { "name": "authenticate-viewer-noncanonical-display-name", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\" Café \"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\" Café \"}", "valid": false }, { "name": "authenticate-viewer-display-name-too-long", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"名名名名名名名名名名名名名名名名名名名名名名名名名\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"名名名名名名名名名名名名名名名名名名名名名名名名名\"}", "valid": false }, { "name": "authenticate-viewer-display-name-at-limit", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"名名名名名名名名名名名名名名名名名名名名名名名名\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"名名名名名名名名名名名名名名名名名名名名名名名名\"}", "valid": true }, { "name": "authenticate-viewer-display-name-bidi-override", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"a‮b\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"a‮b\"}", "valid": false }, { "name": "authenticate-viewer-display-name-newline", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"a\\nb\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"a\\nb\"}", "valid": false }, { "name": "authenticate-viewer-null-display-name", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":null}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":null}", "valid": false }, { "name": "authenticate-unknown-role", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"relay\",\"clientId\":\"client_12345678\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"relay\",\"clientId\":\"client_12345678\"}", "valid": false }, { @@ -283,7 +283,7 @@ { "name": "set-quality-settings-with-audio", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000}}", "valid": true }, { @@ -817,19 +817,19 @@ { "name": "set-quality-settings-folded-audio-invalid", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenaudioquality\":\"invalid\"}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxaudiobitrate\":\"invalid\"}}", "valid": false }, { "name": "set-quality-settings-folded-audio-null", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenaudioquality\":null}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxaudiobitrate\":null}}", "valid": false }, { "name": "set-quality-settings-folded-key-collision", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\",\"screenaudioquality\":\"invalid\"}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000,\"maxaudiobitrate\":\"invalid\"}}", "valid": false }, { @@ -847,25 +847,25 @@ { "name": "authenticate-host-folded-generation", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"sharegeneration\":\"bad\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"sharegeneration\":\"bad\"}", "valid": false }, { "name": "authenticate-host-folded-default-policy", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"routepolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false}}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"routepolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false}}", "valid": false }, { "name": "authenticate-viewer-folded-presence-false", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerpresence\":false}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerpresence\":false}", "valid": false }, { "name": "authenticate-viewer-folded-password-null", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerpassword\":null}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerpassword\":null}", "valid": false }, { @@ -1053,6 +1053,72 @@ "schema": "client", "json": "{\"type\":\"send-room-interaction\",\"requestId\":\"request_12345678\",\"payload\":{\"kind\":\"reaction\",\"reaction\":\"wave\"}}", "valid": true + }, + { + "name": "numeric-audio-6000", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":6000}}", + "valid": true + }, + { + "name": "numeric-audio-96000", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":96000}}", + "valid": true + }, + { + "name": "numeric-audio-256000", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":256000}}", + "valid": true + }, + { + "name": "numeric-audio-320000", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":320000}}", + "valid": true + }, + { + "name": "numeric-audio-510000", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":510000}}", + "valid": true + }, + { + "name": "numeric-audio-5999", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":5999}}", + "valid": false + }, + { + "name": "numeric-audio-510001", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":510001}}", + "valid": false + }, + { + "name": "numeric-audio-128000.5", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000.5}}", + "valid": false + }, + { + "name": "numeric-audio-null", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":null}}", + "valid": false + }, + { + "name": "numeric-audio-music", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":\"music\"}}", + "valid": false + }, + { + "name": "published-1.x-authentication", + "schema": "client", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", + "valid": false } ], "serverMessages": [ @@ -1062,19 +1128,19 @@ }, { "name": "authenticated-host-minimal", - "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v23\",\"peerId\":\"host_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"connectionId\":null,\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"none\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"role\":\"host\",\"viewerPasswordEnabled\":false}" + "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v24\",\"peerId\":\"host_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"connectionId\":null,\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"none\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"role\":\"host\",\"viewerPasswordEnabled\":false}" }, { "name": "authenticated-host-full", - "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v23\",\"peerId\":\"host_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"connectionId\":\"connection_12345678\",\"iceConfig\":{\"iceServers\":[{\"urls\":\"stun:stun.example.test:3478\"},{\"urls\":[\"stun:stun-1.example.test:3478\",\"stun:[2001:db8::1]:3478\"]}],\"natPredictionStunUrls\":[\"stun:nat.example.test:3478\"]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"sfu\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":\"generation_12345678\"},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"role\":\"host\",\"viewerPasswordEnabled\":false}" + "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v24\",\"peerId\":\"host_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"connectionId\":\"connection_12345678\",\"iceConfig\":{\"iceServers\":[{\"urls\":\"stun:stun.example.test:3478\"},{\"urls\":[\"stun:stun-1.example.test:3478\",\"stun:[2001:db8::1]:3478\"]}],\"natPredictionStunUrls\":[\"stun:nat.example.test:3478\"]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"sfu\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":\"generation_12345678\"},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"role\":\"host\",\"viewerPasswordEnabled\":false}" }, { "name": "authenticated-viewer-minimal", - "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v23\",\"peerId\":\"viewer_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"hostPaused\":false,\"connectionId\":null,\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"none\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"role\":\"viewer\"}" + "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v24\",\"peerId\":\"viewer_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"hostPaused\":false,\"connectionId\":null,\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"none\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"role\":\"viewer\"}" }, { "name": "authenticated-viewer-full", - "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v23\",\"peerId\":\"viewer_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"hostPaused\":true,\"connectionId\":\"connection_12345678\",\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"peer\",\"peerId\":\"parent_12345678\"},\"childPeerIds\":[\"child_12345678\",\"child_87654321\"],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"role\":\"viewer\"}" + "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v24\",\"peerId\":\"viewer_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"hostPaused\":true,\"connectionId\":\"connection_12345678\",\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"peer\",\"peerId\":\"parent_12345678\"},\"childPeerIds\":[\"child_12345678\",\"child_87654321\"],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"role\":\"viewer\"}" }, { "name": "signaling-challenge-response", @@ -1130,7 +1196,7 @@ }, { "name": "quality-settings", - "json": "{\"type\":\"quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"}}" + "json": "{\"type\":\"quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000}}" }, { "name": "quality-settings-without-audio", diff --git a/tests/host-peer-tracks.test.ts b/tests/host-peer-tracks.test.ts index c3bed817f..59e812a37 100644 --- a/tests/host-peer-tracks.test.ts +++ b/tests/host-peer-tracks.test.ts @@ -1017,7 +1017,7 @@ describe("HostPeer source replacement", () => { { iceServers: [] }, { ...QUALITY_PROFILES["720p30"], - screenAudioQuality: "very-high", + maxAudioBitrate: 192000, }, ); @@ -1231,6 +1231,26 @@ describe("HostPeer source replacement", () => { expect(connection.senders[1]?.appliedMaxBitrates).toEqual([128_000]); }); + it("applies an answer's audio limit on the existing sender without touching video", async () => { + const peer = createPeer(createStream(createTrack("video", "video"), createTrack("audio", "audio")), + undefined, undefined, { ...QUALITY_PROFILES["720p30"], maxAudioBitrate: 320_000 }); + await peer.start(); + const connection = FakePeerConnection.latest!; + const videoCalls = connection.senders[0]!.setParameters.mock.calls.length; + const audio = connection.senders[1]!; + audio.getParameters().codecs = [{ mimeType: "audio/opus", clockRate: 48000, payloadType: 111, + sdpFmtpLine: "stereo=1;maxaveragebitrate=192000" }]; + await peer.acceptSignal({ kind: "description", connectionId: peer.connectionId, + description: { type: "answer", sdp: "test-answer" } }); + await vi.waitFor(() => expect(audio.getParameters().encodings[0]?.maxBitrate).toBe(192_000)); + expect(connection.senders[0]!.setParameters.mock.calls).toHaveLength(videoCalls); + expect(connection.createOfferCallCount).toBe(1); + expect(peer.getSnapshot().audioSenderParameters).toMatchObject({ + requestedMaxBitrate: 320_000, appliedMaxBitrate: 192_000, mismatch: true, + }); + peer.dispose(); + }); + it("updates quality parameters without replacing media tracks", async () => { const video = createTrack("video", "video"); const audio = createTrack("audio", "audio"); @@ -1257,7 +1277,7 @@ describe("HostPeer source replacement", () => { await expect( peer.updateProfile({ ...QUALITY_PROFILES["1080p60"], - screenAudioQuality: "very-high", + maxAudioBitrate: 192000, }), ).resolves.toBe(true); expect(connection.senders[0]?.setParameters).toHaveBeenCalledTimes(3); @@ -1286,7 +1306,7 @@ describe("HostPeer source replacement", () => { await expect( peer.updateProfile({ ...QUALITY_PROFILES["720p30"], - screenAudioQuality: "very-high", + maxAudioBitrate: 192000, }), ).resolves.toBe(false); @@ -1299,7 +1319,7 @@ describe("HostPeer source replacement", () => { await expect( peer.updateProfile({ ...QUALITY_PROFILES["720p30"], - screenAudioQuality: "very-high", + maxAudioBitrate: 192000, }), ).resolves.toBe(true); expect(audioSender.getParameters().encodings[0]?.maxBitrate).toBe(192_000); @@ -1315,14 +1335,14 @@ describe("HostPeer source replacement", () => { const saver = peer.updateProfile({ ...QUALITY_PROFILES["720p30"], - screenAudioQuality: "saver", + maxAudioBitrate: 64000, }); await vi.waitFor(() => expect(audioSender.setParameters).toHaveBeenCalledTimes(2), ); const veryHigh = peer.updateProfile({ ...QUALITY_PROFILES["720p30"], - screenAudioQuality: "very-high", + maxAudioBitrate: 192000, }); audioSender.releaseDeferredSetParameters(); @@ -3436,7 +3456,7 @@ describe("ViewerRelay downstream ownership", () => { it("applies the latest audio ceiling to current and future children", async () => { const initialProfile = { ...QUALITY_PROFILES["1080p60"], - screenAudioQuality: "saver", + maxAudioBitrate: 64000, } as const; const relay = new ViewerRelay( { iceServers: [] }, @@ -3460,7 +3480,7 @@ describe("ViewerRelay downstream ownership", () => { await expect( relay.updateProfile({ ...initialProfile, - screenAudioQuality: "very-high", + maxAudioBitrate: 192000, }), ).resolves.toBe(true); expect(firstConnection.senders[1]?.appliedMaxBitrates).toEqual([ diff --git a/tests/host-quality-ownership.test.ts b/tests/host-quality-ownership.test.ts index 359a3d44f..ba333b917 100644 --- a/tests/host-quality-ownership.test.ts +++ b/tests/host-quality-ownership.test.ts @@ -4,7 +4,7 @@ import ts from "typescript"; import { describe, expect, it, vi } from "vitest"; import { QUALITY_PROFILES, qualitySettingsEqual, qualitySettingsLabel, - resolveScreenAudioQuality, videoQualitySettingsEqual, + screenAudioBitrate, videoQualitySettingsEqual, type QualitySettings, } from "../src/client/media/quality"; import { NativeSenderPeer } from "../src/client/native/native-sender-peer"; @@ -108,7 +108,7 @@ function fixture(launchedByClient = true) { setDetails: vi.fn(), setNativeActive: vi.fn(), applyCaptureProfile: vi.fn(async (_stream: unknown, profile: QualitySettings) => { physical = profile; }), captureDetails: () => ({}), - qualitySettingsEqual, videoQualitySettingsEqual, resolveScreenAudioQuality, qualitySettingsLabel, + qualitySettingsEqual, videoQualitySettingsEqual, screenAudioBitrate, qualitySettingsLabel, say: (key: string) => key, syncHostSfuQualityWarning: () => null, NativeSenderPeer, discardPreparedHostChild: vi.fn(), removePeer: vi.fn(), startPeer: vi.fn(async () => undefined), nativePreviewTailRef: ref(Promise.resolve()), setSwitchingSource: vi.fn(), diff --git a/tests/native-client.test.ts b/tests/native-client.test.ts index b76fc626a..65dd2071b 100644 --- a/tests/native-client.test.ts +++ b/tests/native-client.test.ts @@ -216,7 +216,7 @@ describe("native App private wire", () => { }); const fetcher = vi.fn(async (url: string) => url.endsWith("/health") ? new Response(JSON.stringify({ - protocol: 9, service: "piik-client", port: 39_721, + protocol: 10, service: "piik-client", port: 39_721, instanceToken: "a".repeat(43), nativeMedia: {video: true, processAudio: false, systemAudio: true, hardwareH264: true, softwareVP8: true}, }), {status: 200}) @@ -243,7 +243,7 @@ describe("native App private wire", () => { static readonly CLOSING = 2; static readonly CLOSED = 3; readyState = FakeWebSocket.OPEN; - protocol = `piik-client-v9.${token}`; + protocol = `piik-client-v10.${token}`; readonly close = vi.fn(() => { this.readyState = FakeWebSocket.CLOSING; }); @@ -261,7 +261,7 @@ describe("native App private wire", () => { const event = new Event("message"); Object.defineProperty(event, "data", { value: JSON.stringify({ - version: 9, + version: 10, id: request.id, type: request.type === "hello" ? "ready" : request.type === "ping" ? "pong" : "request-failed", ...(request.type === "update-share" ? { code: "operation-failed" } : {}), @@ -283,7 +283,7 @@ describe("native App private wire", () => { }); vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ - protocol: 9, + protocol: 10, service: "piik-client", port: 39_721, instanceToken: token, @@ -336,7 +336,7 @@ describe("native App private wire", () => { it("accepts descriptive health extensions and treats absent media features as unavailable", () => { expect( nativeHealthSchema.parse({ - protocol: 9, + protocol: 10, service: "piik-client", port: 39_721, instanceToken: "a".repeat(43), @@ -388,7 +388,7 @@ describe("native App private wire", () => { static readonly OPEN = 1; static readonly CLOSING = 2; readyState = CaptureSocket.OPEN; - protocol = `piik-client-v9.${health.instanceToken}`; + protocol = `piik-client-v10.${health.instanceToken}`; constructor() { super(); queueMicrotask(() => this.dispatchEvent(new Event("open"))); @@ -444,7 +444,7 @@ describe("native App private wire", () => { static readonly OPEN = 1; static readonly CLOSING = 2; readyState = Socket.OPEN; - protocol = `piik-client-v9.${health.instanceToken}`; + protocol = `piik-client-v10.${health.instanceToken}`; constructor() { super(); queueMicrotask(() => this.dispatchEvent(new Event("open"))); } close() { this.readyState = Socket.CLOSING; } send(payload: string) { @@ -493,11 +493,11 @@ describe("native App private wire", () => { static readonly OPEN = 1; static readonly CLOSING = 2; readyState = Socket.OPEN; - protocol = `piik-client-v9.${health.instanceToken}`; + protocol = `piik-client-v10.${health.instanceToken}`; constructor() { super(); socket = this; queueMicrotask(() => this.dispatchEvent(new Event("open"))); } close() { this.readyState = Socket.CLOSING; } ack(request: Record, type: string, fields = {}) { - this.dispatchEvent(new MessageEvent("message", { data: JSON.stringify({ version: 9, id: request.id, type, ...fields }) })); + this.dispatchEvent(new MessageEvent("message", { data: JSON.stringify({ version: 10, id: request.id, type, ...fields }) })); } send(payload: string) { const request = JSON.parse(payload) as Record; @@ -556,14 +556,14 @@ describe("native App private wire", () => { static readonly OPEN = 1; static readonly CLOSING = 2; readyState = Socket.OPEN; - protocol = `piik-client-v9.${health.instanceToken}`; + protocol = `piik-client-v10.${health.instanceToken}`; constructor() { super(); queueMicrotask(() => this.dispatchEvent(new Event("open"))); } close() { this.readyState = Socket.CLOSING; } send(payload: string) { const request = JSON.parse(payload) as Record; requests.push(request); queueMicrotask(() => this.dispatchEvent(new MessageEvent("message", { data: JSON.stringify({ - version: 9, id: request.id, ...(request.type === "hello" ? { type: "ready" } : { + version: 10, id: request.id, ...(request.type === "hello" ? { type: "ready" } : { type: "receive-answer", shareId: request.shareId, connectionId: request.connectionId, sdp: "answer", audio: false, codec: "vp8", ...(supported ? { reused: true } : {}), }), @@ -578,7 +578,7 @@ describe("native App private wire", () => { const client = (await NativeClient.connect())!; try { const result = await client.receiveOffer("share_123456", "connection_1234", { type: "offer", sdp: "offer" }, { iceServers: [] }, 2); - expect(requests.at(-1)).toEqual({ version: 9, id: expect.any(String), type: "receive-offer", + expect(requests.at(-1)).toEqual({ version: 10, id: expect.any(String), type: "receive-offer", shareId: "share_123456", connectionId: "connection_1234", sdp: "offer", iceServers: [], edgeCapacity: 2, ...(supported ? { reuseReceiver: true } : {}), }); @@ -623,7 +623,7 @@ describe("native App private wire", () => { it("fences native events by share and connection identity", () => { const event = { - version: 9, + version: 10, type: "edge-state", shareId: "share_123456", connectionId: "edge_1234567", @@ -637,7 +637,7 @@ describe("native App private wire", () => { nativeEventSchema.safeParse({ ...event, routeRevision: 1 }).success, ).toBe(false); const path = { - version: 9, + version: 10, type: "edge-path", shareId: "share_123456", connectionId: "edge_1234567", @@ -652,7 +652,7 @@ describe("native App private wire", () => { it("keeps an unavailable preview advisory instead of treating it as media failure", () => { const preview = { - version: 9, + version: 10, id: "request_preview", type: "source-preview", sourceKey: "display:65537", @@ -676,10 +676,10 @@ describe("native App private wire", () => { maxFramerate: 60, maxBitrate: 12_000_000, degradationPreference: "maintain-framerate", - screenAudioQuality: "very-high", + maxAudioBitrate: 192000, }; expect(shareStartedResponseSchema.safeParse({ - version: 9, + version: 10, id: "request_start", type: "share-started", shareId: "share_123456", @@ -687,19 +687,19 @@ describe("native App private wire", () => { codec: "h264", }).success).toBe(true); expect(shareUpdatedResponseSchema.safeParse({ - version: 9, + version: 10, id: "request_update", type: "share-updated", shareId: "share_123456", }).success).toBe(true); expect(shareSourceReplacedResponseSchema.safeParse({ - version: 9, + version: 10, id: "request_source", type: "share-source-replaced", shareId: "share_123456", }).success).toBe(true); expect(shareUpdatedResponseSchema.safeParse({ - version: 9, + version: 10, id: "request_update", type: "share-updated", shareId: "share_123456", @@ -709,7 +709,7 @@ describe("native App private wire", () => { it("requires the actual Native codec instead of reporting Auto as media", () => { const response = { - version: 9, + version: 10, id: "request_receive", type: "receive-answer", shareId: "share_123456", @@ -720,7 +720,7 @@ describe("native App private wire", () => { for (const codec of ["h264", "vp8"]) { expect(receiveAnswerResponseSchema.safeParse({ ...response, codec }).success).toBe(true); expect(shareStartedResponseSchema.safeParse({ - version: 9, id: "request_start", type: "share-started", + version: 10, id: "request_start", type: "share-started", shareId: response.shareId, audio: false, codec, }).success).toBe(true); } @@ -730,7 +730,7 @@ describe("native App private wire", () => { it("accepts only internally consistent native quality evidence", () => { const event = { - version: 9, + version: 10, type: "edge-quality", shareId: "share_123456", connectionId: "edge_1234567", diff --git a/tests/native-media-bridge.test.ts b/tests/native-media-bridge.test.ts index dabb9c1a9..9bf8eb9b7 100644 --- a/tests/native-media-bridge.test.ts +++ b/tests/native-media-bridge.test.ts @@ -80,7 +80,7 @@ function fixture(options: { NativeMediaBridgeControl["prepareLocalEdge"] >(async (_shareId, connectionId) => { listener?.({ - version: 9, + version: 10, type: "edge-candidate", shareId: "share_123456", connectionId, @@ -185,7 +185,7 @@ describe("native media bridge", () => { current.peer().dispatchEvent(Object.assign(new Event("icecandidate"), { candidate: null })); } else { current.peer().addIceCandidate.mockRejectedValueOnce(cause); - current.emit({ version: 9, type: "edge-candidate", shareId: "share_123456", + current.emit({ version: 10, type: "edge-candidate", shareId: "share_123456", connectionId: current.bridge.connectionId, candidate: null }); } await vi.waitFor(() => expect(current.onFailed).toHaveBeenCalledOnce()); @@ -205,7 +205,7 @@ describe("native media bridge", () => { const result = starting.catch((error: unknown) => error); if (phase === "queued") { current.emit({ - version: 9, type: "edge-candidate", shareId: "share_123456", + version: 10, type: "edge-candidate", shareId: "share_123456", connectionId: current.bridge.connectionId, candidate: { candidate: "candidate:2 1 udp 1 127.0.0.1 10 typ host" }, }); @@ -218,13 +218,13 @@ describe("native media bridge", () => { if (phase === "active") { peer.addIceCandidate.mockRejectedValueOnce(new DOMException("Rejected candidate", "OperationError")); current.emit({ - version: 9, type: "edge-candidate", shareId: "share_123456", + version: 10, type: "edge-candidate", shareId: "share_123456", connectionId: current.bridge.connectionId, candidate: { candidate: "candidate:2 1 udp 1 127.0.0.1 10 typ host" }, }); } current.emit({ - version: 9, type: "edge-candidate", shareId: "share_123456", + version: 10, type: "edge-candidate", shareId: "share_123456", connectionId: current.bridge.connectionId, candidate: null, }); await vi.waitFor(() => expect(debugError).toHaveBeenCalledWith( @@ -378,14 +378,14 @@ describe("native media bridge", () => { await starting; current.emit({ - version: 9, + version: 10, type: "edge-state", shareId: "share_123456", connectionId: current.bridge.connectionId, state: "failed", }); current.emit({ - version: 9, + version: 10, type: "edge-state", shareId: "share_123456", connectionId: current.bridge.connectionId, diff --git a/tests/native-sender-edge.test.ts b/tests/native-sender-edge.test.ts index f095c7585..4abe7b015 100644 --- a/tests/native-sender-edge.test.ts +++ b/tests/native-sender-edge.test.ts @@ -66,17 +66,17 @@ describe("native sender edge adapter", () => { "keeps connected media when trickle signaling is unavailable: %j", async (candidate) => { const current = fixture(); expect(await current.edge.start()).toBe(true); - current.emit({ version: 9, type: "edge-state", shareId: "share_1234567", connectionId: "edge_12345678", state: "connected" }); + current.emit({ version: 10, type: "edge-state", shareId: "share_1234567", connectionId: "edge_12345678", state: "connected" }); current.sendSignal.mockReturnValue(false); - current.emit({ version: 9, type: "edge-candidate", shareId: "share_1234567", connectionId: "edge_12345678", candidate }); + current.emit({ version: 10, type: "edge-candidate", shareId: "share_1234567", connectionId: "edge_12345678", candidate }); expect(current.edge.isConnected()).toBe(true); expect(current.control.closeEdge).not.toHaveBeenCalled(); expect(current.states).toEqual(["connected"]); - current.emit({ version: 9, type: "edge-state", shareId: "share_1234567", connectionId: "edge_12345678", state: "failed" }); + current.emit({ version: 10, type: "edge-state", shareId: "share_1234567", connectionId: "edge_12345678", state: "failed" }); expect(current.edge.isConnected()).toBe(false); expect(current.states).toEqual(["connected", "failed"]); current.edge.dispose(); - current.emit({ version: 9, type: "edge-state", shareId: "share_1234567", connectionId: "edge_12345678", state: "connected" }); + current.emit({ version: 10, type: "edge-state", shareId: "share_1234567", connectionId: "edge_12345678", state: "connected" }); expect(current.edge.isConnected()).toBe(false); expect(current.control.closeEdge).toHaveBeenCalledOnce(); }, @@ -90,7 +90,7 @@ describe("native sender edge adapter", () => { const current = fixture(); vi.mocked(current.control.prepareEdge).mockImplementationOnce(async () => { - current.emit({ version: 9, type: "edge-candidate", shareId: "share_1234567", connectionId: "edge_12345678", candidate: null }); + current.emit({ version: 10, type: "edge-candidate", shareId: "share_1234567", connectionId: "edge_12345678", candidate: null }); return { type: "offer", sdp: "v=0\r\n" }; }); current.sendSignal.mockImplementation((_peer, payload) => payload.kind === "description"); @@ -104,7 +104,7 @@ describe("native sender edge adapter", () => { const prepare = vi.mocked(current.control.prepareEdge); prepare.mockImplementationOnce(async () => { current.emit({ - version: 9, + version: 10, type: "edge-candidate", shareId: "share_1234567", connectionId: "edge_12345678", @@ -125,7 +125,7 @@ describe("native sender edge adapter", () => { const current = fixture(); expect(await current.edge.start()).toBe(true); current.emit({ - version: 9, + version: 10, type: "edge-state", shareId: "share_1234567", connectionId: "edge_12345678", @@ -153,7 +153,7 @@ describe("native sender edge adapter", () => { .toHaveLength(3); for (const [index, port] of [40_000, 40_003, 40_006].entries()) { current.emit({ - version: 9, + version: 10, type: "edge-candidate", shareId: "share_1234567", connectionId: "edge_12345678", diff --git a/tests/native-sender-peer.test.ts b/tests/native-sender-peer.test.ts index 439a122a3..91b8123b4 100644 --- a/tests/native-sender-peer.test.ts +++ b/tests/native-sender-peer.test.ts @@ -67,7 +67,7 @@ describe("native Host peer quality", () => { qualityProbe: true, })).toBe(false); listener({ - version: 9, + version: 10, type: "edge-state", shareId: "share_1234567", connectionId: "edge_12345678", @@ -75,7 +75,7 @@ describe("native Host peer quality", () => { }); const quality = { - version: 9 as const, + version: 10 as const, type: "edge-quality" as const, shareId: "share_1234567", connectionId: "edge_12345678", diff --git a/tests/native-sfu-publisher.test.ts b/tests/native-sfu-publisher.test.ts index 31f57297f..5412b8768 100644 --- a/tests/native-sfu-publisher.test.ts +++ b/tests/native-sfu-publisher.test.ts @@ -30,7 +30,7 @@ describe("Native SFU publication", () => { const activation = publisher.activate(); if (mode === "transport") { await activation; - listener!({ version: 9, type: "publication-state", shareId: "native_share", ...config, state: "failed" }); + listener!({ version: 10, type: "publication-state", shareId: "native_share", ...config, state: "failed" }); } const firstClose = publisher.disconnect(); expect(publisher.disconnect()).toBe(firstClose); @@ -65,7 +65,7 @@ describe("Native SFU publication", () => { const control = { preparePublication: vi.fn(async () => { emit({ - version: 9, + version: 10, type: "publication-candidate", shareId: "native_share", ...config, @@ -135,7 +135,7 @@ describe("Native SFU publication", () => { expect(await publisher.updateProfile()).toBe(true); expect(sent.at(-1)).toMatchObject({ kind: "media", media }); emit({ - version: 9, + version: 10, type: "publication-quality", shareId: "native_share", ...config, @@ -170,12 +170,12 @@ describe("Native SFU publication", () => { frameHeight: 360, }), ); - emit({ version: 9, type: "publication-state", shareId: "native_share", ...config, state: "disconnected" }); + emit({ version: 10, type: "publication-state", shareId: "native_share", ...config, state: "disconnected" }); expect(control.closePublication).not.toHaveBeenCalled(); expect(onDisconnected).not.toHaveBeenCalled(); - emit({ version: 9, type: "publication-state", shareId: "native_share", ...config, state: "connected" }); + emit({ version: 10, type: "publication-state", shareId: "native_share", ...config, state: "connected" }); emit({ - version: 9, + version: 10, type: "publication-state", shareId: "native_share", ...config, diff --git a/tests/native-viewer-peer.test.ts b/tests/native-viewer-peer.test.ts index 89f26a8fc..d0d42eac7 100644 --- a/tests/native-viewer-peer.test.ts +++ b/tests/native-viewer-peer.test.ts @@ -94,7 +94,7 @@ it.each(["viewer", "route"] as const)("retires a connected Native receiver on co description: { type: "offer", sdp: "v=0\r\nm=video 9 UDP/TLS/RTP/SAVPF 96\r\na=sendonly\r\na=rtpmap:96 VP8/90000\r\n" }, }); const connected = { - version: 9, shareId: "session", connectionId: "current", type: "edge-state", state: "connected", + version: 10, shareId: "session", connectionId: "current", type: "edge-state", state: "connected", } as const; onEvent(connected); expect(peer.isConnected()).toBe(true); @@ -215,7 +215,7 @@ it.each(["reused", "replaced", "answer-lost", "control-lost"])("keeps Native rec }); try { await peer.acceptSignal("parent", offer("first")); - event({ version: 9, type: "edge-state", shareId: "session", connectionId: "current", state: "connected" }); + event({ version: 10, type: "edge-state", shareId: "session", connectionId: "current", state: "connected" }); const source = peer.nativeSource; receiveOffer.mockImplementationOnce(async () => { if (outcome === "replaced") { @@ -567,7 +567,7 @@ it.each(["first", "replacement", "failed-offer"])("keeps committed route recover const offer = (connectionId: string) => ({ kind: "description" as const, connectionId, description: { type: "offer" as const, sdp: "v=0\r\nm=video 9 UDP/TLS/RTP/SAVPF 96\r\na=sendonly\r\na=rtpmap:96 VP8/90000\r\n" }}); const state = (connectionId: string, value: "connected" | "failed") => onEvent({ - version: 9, shareId: "session", connectionId, type: "edge-state", state: value, + version: 10, shareId: "session", connectionId, type: "edge-state", state: value, }); try { await peer.acceptSignal("parent", offer("first")); @@ -589,7 +589,7 @@ it.each(["first", "replacement", "failed-offer"])("keeps committed route recover const retiredEvent = onEvent; if (scenario === "failed-offer") receiveOffer.mockRejectedValueOnce(new Error("receiver unavailable")); await peer.acceptSignal("parent", offer("rebuilt")); - retiredEvent({ version: 9, shareId: "session", connectionId, type: "edge-state", state: "connected" }); + retiredEvent({ version: 10, shareId: "session", connectionId, type: "edge-state", state: "connected" }); state("rebuilt", "failed"); expect(restart).toHaveBeenCalledTimes(2); expect(exhausted).toHaveBeenCalledExactlyOnceWith("parent", "rebuilt"); diff --git a/tests/protocol.test.ts b/tests/protocol.test.ts index 28ce6b697..c5c7228b9 100644 --- a/tests/protocol.test.ts +++ b/tests/protocol.test.ts @@ -87,7 +87,7 @@ describe("runtime capabilities", () => { const qualitySettingsWithAudio = { ...qualitySettings, - screenAudioQuality: "music", + maxAudioBitrate: 128000, } as const; const legacyCodecQualitySettings = { @@ -139,7 +139,7 @@ const qualityEvidence = { describe("client signaling protocol", () => { it("uses the current strict signaling generation", () => { - expect(SIGNALING_PROTOCOL).toBe("piik-v23"); + expect(SIGNALING_PROTOCOL).toBe("piik-v24"); }); it("keeps signaling challenges strict and sequence-only", () => { @@ -772,15 +772,13 @@ describe("client signaling protocol", () => { qualitySettings: { ...qualitySettings, resolution: "480p" }, }).success, ).toBe(true); - for (const screenAudioQuality of [ - "saver", - "music", - "very-high", + for (const maxAudioBitrate of [ + 6_000, 96_000, 256_000, 320_000, 510_000, ] as const) { expect( clientMessageSchema.safeParse({ type: "set-quality-settings", - qualitySettings: { ...qualitySettings, screenAudioQuality }, + qualitySettings: { ...qualitySettings, maxAudioBitrate }, }).success, ).toBe(true); } @@ -794,8 +792,14 @@ describe("client signaling protocol", () => { { ...qualitySettings, maxBitrate: 5_000_000.5 }, { ...qualitySettings, degradationPreference: "automatic" }, legacyCodecQualitySettings, - { ...qualitySettings, screenAudioQuality: "lossless" }, - { ...qualitySettings, screenAudioQuality: 960_000 }, + { ...qualitySettings, maxAudioBitrate: "lossless" }, + { ...qualitySettings, maxAudioBitrate: "music" }, + { ...qualitySettings, maxAudioBitrate: 0 }, + { ...qualitySettings, maxAudioBitrate: 5_999 }, + { ...qualitySettings, maxAudioBitrate: 510_001 }, + { ...qualitySettings, maxAudioBitrate: 128_000.5 }, + { ...qualitySettings, maxAudioBitrate: null }, + { ...qualitySettings, maxAudioBitrate: 960_000 }, { ...qualitySettings, codec: "video/VP9" }, ]) { expect( diff --git a/tests/quality-preference.test.ts b/tests/quality-preference.test.ts index 282968565..a279e3f09 100644 --- a/tests/quality-preference.test.ts +++ b/tests/quality-preference.test.ts @@ -14,7 +14,7 @@ describe("sharing quality preference", () => { expect(readPreferredQuality()).toEqual(DEFAULT_QUALITY_SETTINGS); const chosen = { ...DEFAULT_QUALITY_SETTINGS, resolution: "1440p" as const, maxFramerate: 60, maxBitrate: 10_000_000, degradationPreference: "maintain-resolution" as const, - screenAudioQuality: "very-high" as const }; + maxAudioBitrate: 256_000 }; savePreferredQuality(chosen); expect(readPreferredQuality()).toEqual(chosen); savePreferredQuality({ ...chosen, maxBitrate: -1 }); diff --git a/tests/quality.test.ts b/tests/quality.test.ts index 12000d778..d0b435f65 100644 --- a/tests/quality.test.ts +++ b/tests/quality.test.ts @@ -13,6 +13,7 @@ import { QUALITY_RESOLUTIONS, qualitySettingsEqual, screenAudioBitrate, + screenAudioSenderCeiling, senderParameterWarning, setMediaPaused, startupVideoProfile, @@ -150,10 +151,10 @@ describe("realtime quality controls", () => { it("keeps audio selection orthogonal while defaulting missing settings to music", () => { const saver = { ...QUALITY_PROFILES["1080p60"], - screenAudioQuality: "saver", + maxAudioBitrate: 64000, } as const; const { - screenAudioQuality: _screenAudioQuality, + maxAudioBitrate: _maxAudioBitrate, ...legacySettings } = QUALITY_PROFILES["1080p60"]; @@ -295,9 +296,9 @@ describe("realtime quality controls", () => { it.each([ [undefined, 128_000], - ["saver", 64_000], - ["music", 128_000], - ["very-high", 192_000], + [64_000, 64_000], + [128_000, 128_000], + [192_000, 192_000], ] as const)("applies and reads back the %s audio ceiling", async (quality, bitrate) => { let applied = { encodings: [] } as unknown as RTCRtpSendParameters; const sender = { @@ -331,6 +332,38 @@ describe("realtime quality controls", () => { }); }); + it.each([192_000, 510_000])("honors the receiver's %i Opus limit on every live update", async (limit) => { + let applied = { + codecs: [{ mimeType: "audio/opus", sdpFmtpLine: `stereo=1;maxaveragebitrate=${limit}` }], + encodings: [{}], + } as RTCRtpSendParameters; + const sender = { + getParameters: () => applied, + setParameters: async (value: RTCRtpSendParameters) => { applied = value; }, + } as RTCRtpSender; + for (const bitrate of [128_000, 320_000, 96_000, 510_000]) { + const ceiling = Math.min(bitrate, limit, 320_000); + await expect(configureScreenAudioSender(sender, bitrate)).resolves.toEqual({ + requestedMaxBitrate: bitrate, appliedMaxBitrate: ceiling, mismatch: ceiling !== bitrate, + }); + expect(applied.encodings[0]?.maxBitrate).toBe(ceiling); + } + }); + + it.each([ + ["", 32_000], + ["stereo=1", 64_000], + ["stereo=1;maxaveragebitrate=invalid", 64_000], + ["stereo=1;maxaveragebitrate=510001", 64_000], + ["maxplaybackrate=8000", 12_000], + ["stereo=1;maxplaybackrate=16000", 40_000], + ["maxaveragebitrate=510000;maxaveragebitrate=96000", 96_000], + ] as const)("keeps omitted/invalid Opus limits bounded: %s", (sdpFmtpLine, expected) => { + const parameters = { codecs: [{ mimeType: "audio/opus", sdpFmtpLine }] } as RTCRtpSendParameters; + expect(screenAudioSenderCeiling(parameters, 320_000)).toBe(expected); + expect(screenAudioSenderCeiling(parameters, 6_000)).toBe(6_000); + }); + it("reports an audio ceiling the browser rewrites", async () => { const sender = { getParameters: vi diff --git a/tests/screen-audio-sdp.test.ts b/tests/screen-audio-sdp.test.ts index fb4850157..c9893c9f8 100644 --- a/tests/screen-audio-sdp.test.ts +++ b/tests/screen-audio-sdp.test.ts @@ -35,7 +35,7 @@ describe("preferScreenAudioStereo", () => { expect(session.media[0]?.fmtp).toContainEqual({ payload: 111, config: - "minptime=10;useinbandfec=1;x-stereo=keep;stereo=1;maxaveragebitrate=192000", + "minptime=10;useinbandfec=1;x-stereo=keep;stereo=1;maxaveragebitrate=510000", }); expect(session.media[1]?.fmtp).toEqual([ { payload: 96, config: "packetization-mode=1;profile-level-id=42e01f" }, @@ -52,7 +52,7 @@ describe("preferScreenAudioStereo", () => { }); expect(parse(result.sdp!).media[0]?.fmtp[0]?.config).toBe( - "stereo=1;x-stereo=0;maxaveragebitrate=192000;x-maxaveragebitrate=keep", + "stereo=1;x-stereo=0;maxaveragebitrate=510000;x-maxaveragebitrate=keep", ); }); @@ -64,7 +64,7 @@ describe("preferScreenAudioStereo", () => { expect(parse(result.sdp!).media[0]?.fmtp).toContainEqual({ payload: 111, - config: "stereo=1;maxaveragebitrate=192000", + config: "stereo=1;maxaveragebitrate=510000", }); }); @@ -82,7 +82,7 @@ describe("preferScreenAudioStereo", () => { expect(audio[0]?.fmtp[0]?.config).toBe("stereo=0"); expect(audio[1]?.fmtp[0]?.config).toContain("stereo=1"); - expect(audio[1]?.fmtp[0]?.config).toContain("maxaveragebitrate=192000"); + expect(audio[1]?.fmtp[0]?.config).toContain("maxaveragebitrate=510000"); }); it.each([ diff --git a/tests/sfu-client.test.ts b/tests/sfu-client.test.ts index 3beaaa58b..fdce7d900 100644 --- a/tests/sfu-client.test.ts +++ b/tests/sfu-client.test.ts @@ -373,7 +373,7 @@ describe("embedded SFU browser transport", () => { const { publisher: host, pc, send } = await publisher(); const profile = { ...QUALITY_PROFILES["720p30"], - screenAudioQuality: "very-high" as const, + maxAudioBitrate: 192000 as const, }; expect(await host.updateProfile(profile)).toBe(true); expect(pc.transceivers[0]!.sender.parameters.encodings).toEqual([ @@ -680,7 +680,7 @@ describe("embedded SFU browser transport", () => { expect( await host.updateProfile({ ...QUALITY_PROFILES["1080p30"], - screenAudioQuality: "saver", + maxAudioBitrate: 64000, }), ).toBe(true); expect(video.clones[0]!.applyConstraints).toHaveBeenCalledTimes( diff --git a/tests/viewer-peer-generation.test.ts b/tests/viewer-peer-generation.test.ts index 9f40f2f6e..fcef6b920 100644 --- a/tests/viewer-peer-generation.test.ts +++ b/tests/viewer-peer-generation.test.ts @@ -455,7 +455,7 @@ describe("ViewerPeer connection generations", () => { expect(description).toEqual(expect.objectContaining({ type: "answer", sdp: expect.stringContaining( - "a=fmtp:111 minptime=10;useinbandfec=1;stereo=1;maxaveragebitrate=192000", + "a=fmtp:111 minptime=10;useinbandfec=1;stereo=1;maxaveragebitrate=510000", ), })); expect(answers[index]?.description.sdp).toBe(description.sdp); From 87b608c50f450bc1c6ad7deae90019326b9bd748 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:53:31 +0800 Subject: [PATCH 067/262] fix(media): declare Opus receive capacity on native preview --- docs/research/browser-screen-audio-quality.md | 3 ++- src/client/native/media-bridge.ts | 3 ++- tests/native-media-bridge.test.ts | 25 +++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/research/browser-screen-audio-quality.md b/docs/research/browser-screen-audio-quality.md index 02341fbcc..153c7a605 100644 --- a/docs/research/browser-screen-audio-quality.md +++ b/docs/research/browser-screen-audio-quality.md @@ -110,7 +110,8 @@ networks, mobile playback and physical listening remain open. RFC 7587 gives an omitted receive bitrate a mode-dependent default, not an unrestricted value, and asks receivers to ignore out-of-range preferences. -All candidate Piik receivers therefore declare the full Opus range explicitly. +All candidate Piik receivers therefore declare the full Opus range explicitly, +including the App's local preview bridge through the same Browser answer helper. Browser sender enforcement uses [libwebrtc's existing default calculation](https://webrtc.googlesource.com/src/+/refs/heads/main/modules/audio_coding/codecs/opus/audio_encoder_opus.cc) when a peer omits a valid limit. Unchanged forwarding requires an explicit sufficient limit; it cannot turn down the shared source for one receiver. diff --git a/src/client/native/media-bridge.ts b/src/client/native/media-bridge.ts index 6d181326c..dcaf209f5 100644 --- a/src/client/native/media-bridge.ts +++ b/src/client/native/media-bridge.ts @@ -4,6 +4,7 @@ import { browserDebugEnabled, debugError, debugEvent } from "../lib/debug"; import { observeDebugConnection } from "../lib/debug-webrtc"; import type { ConnectionMetrics } from "../types"; import { addRemoteIceCandidate, type SignalCandidate } from "../webrtc/nat-prediction"; +import { preferScreenAudioStereo } from "../webrtc/screen-audio-sdp"; import { collectConnectionMetrics, decodedVideoFrames, @@ -183,7 +184,7 @@ export class NativeMediaBridge { for (const candidate of this.pendingCandidates.splice(0)) { await addRemoteIceCandidate(this.peer, candidate); } - const answer = await this.peer.createAnswer(); + const answer = preferScreenAudioStereo(await this.peer.createAnswer()); if (!answer.sdp) { throw new Error("Native media bridge produced no SDP answer"); } diff --git a/tests/native-media-bridge.test.ts b/tests/native-media-bridge.test.ts index 9bf8eb9b7..da1273ee7 100644 --- a/tests/native-media-bridge.test.ts +++ b/tests/native-media-bridge.test.ts @@ -130,6 +130,31 @@ afterEach(() => { }); describe("native media bridge", () => { + it("advertises the same Opus receive range on local preview as remote viewers", async () => { + const current = fixture(); + current.peer().createAnswer.mockResolvedValueOnce({ type: "answer", sdp: + "v=0\r\no=- 1 1 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\n" + + "m=audio 9 UDP/TLS/RTP/SAVPF 111\r\na=rtpmap:111 opus/48000/2\r\n" + + "a=fmtp:111 minptime=10;useinbandfec=1\r\n", + }); + const starting = current.bridge.start(true); + await vi.waitFor(() => expect(current.control.acceptSignal).toHaveBeenCalledWith( + "share_123456", current.bridge.connectionId, expect.objectContaining({ + kind: "description", description: expect.objectContaining({ + sdp: expect.stringContaining("stereo=1;maxaveragebitrate=510000"), + }), + }), + )); + expect(current.peer().setLocalDescription).toHaveBeenCalledWith(expect.objectContaining({ + sdp: expect.stringContaining("stereo=1;maxaveragebitrate=510000"), + })); + current.peer().emitTrack({ kind: "video", stop: vi.fn() } as unknown as MediaStreamTrack); + current.peer().emitTrack({ kind: "audio", stop: vi.fn() } as unknown as MediaStreamTrack); + current.peer().setState("connected"); + await expect(starting).resolves.toBe(current.bridge.stream); + current.bridge.dispose(); + }); + it("releases an unstarted Browser reservation without sending an edge command", async () => { const current = fixture(); current.bridge.dispose(); From 1a7c7d70fc5d9a0f4e82bf8d0e72254501336b7c Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:59:55 +0800 Subject: [PATCH 068/262] fix(capture): preserve source geometry within Windows output ceilings --- docs/research/native-client-media.md | 34 +++++++++++++-- docs/standards/media-quality.md | 4 ++ docs/status.md | 4 +- docs/todo.md | 5 ++- docs/verification-status.md | 3 +- native/capture/windows/capture_geometry.h | 15 +++++++ .../capture/windows/capture_geometry.test.cpp | 14 +++++++ native/capture/windows/main.cpp | 5 ++- native/capture/windows/output_worker.test.cpp | 34 +++++++++++++++ scripts/app-media-gate.ts | 41 +++++++++++-------- scripts/app-native-host-gate.ts | 40 +++++++++++------- scripts/embedded-sfu-gate.ts | 16 ++++---- scripts/embedded-sfu-page.ts | 7 +++- scripts/media-gate-geometry.ts | 11 +++++ 14 files changed, 186 insertions(+), 47 deletions(-) create mode 100644 scripts/media-gate-geometry.ts diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index adffac81f..82508f7c0 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -519,17 +519,43 @@ bounds dimensions. Native capture, forwarding and SFU declarations also admit specific encoded formats. Removing one validator would leave these consumers inconsistent. -Source review also distinguishes output geometry from a quality limit. Windows -`FrameConverter` fits content into the selected landscape raster with a black -background, so its padding is encoded. Browser display capture instead requests +Source review also distinguishes output geometry from a quality limit. The +published Windows path fits content into the selected landscape raster with a +black background, so its padding is encoded. Browser display capture instead requests landscape width/height ceilings; a portrait source can be reduced unnecessarily by the height ceiling. The shared TV uses `object-fit: contain` in a 16:9 stage, adding presentation space independently of encoded padding. For example, the -Windows fit rule places 1080x1920 content at 608x1080 inside a 1920x1080 output. +published Windows fit rule places 1080x1920 content at 608x1080 inside a 1920x1080 output. This is a code-derived result, not a new physical capture measurement. Evaluate orientation-independent limits and preservation of source aspect ratio together; adding numeric fields alone would retain the current geometry limitation. +The candidate first changes the existing Windows output worker: it passes the +source's fitted even dimensions to WebRTC, retaining the selected profile as a +ceiling. A 640x480 source stays 640x480; portrait and ultrawide sources no longer +require an encoded landscape canvas. The converter, adaptation owner, output +generation and frame envelope are reused. Axis-specific limits still apply; +this does not yet give portrait capture a transposed ceiling or add a 4K setting. +Browser and other platform geometry remain separate pending work. + +Local synthetic-input controls exercised that production worker through four +successive source sizes: 1080x1920, 3440x1440, 640x480 and 1920x1080. Under the +1080p ceiling, H.264 and VP8 each produced 40 frames at 606x1080, 1920x802, +640x480 and 1920x1080; Chrome decoded every frame at the declared dimensions. +Separately, a codec-only control bypassed product profile admission and encoded +1080x1920, 3440x1440 and 3840x2160 at 30 fps with the current Windows H.264 +encoder. Chrome decoded all 30 frames per size; observed levels were 4.0, 5.0 +and 5.1. This establishes local codec feasibility, not product 4K support, +4K60, sustained performance or cross-device acceptance. + +The real Windows H.264 source also passed the existing App UI and two-Viewer +gates: live/paused settings, background/window recovery, source replacement and +retirement retained their expected owners. Native SFU passed 300 decoded frames +at its original source aspect, reduction from 658x720 to 438x480, numeric audio +changes, Viewer reload/recovery and cleanup. Gate expectations now derive from +the observed source and selected ceilings, instead of requiring encoded 16:9 +padding. Physical game, macOS/Linux and orientation-change acceptance remain open. + The restriction extends beyond resolution labels: shared readers also bound frame rate and bitrate, and Native's selectable profile admits four exact raster pairs while derived outputs admit other dimensions within axis-specific limits. diff --git a/docs/standards/media-quality.md b/docs/standards/media-quality.md index 980d0b8ec..2ab476242 100644 --- a/docs/standards/media-quality.md +++ b/docs/standards/media-quality.md @@ -42,6 +42,10 @@ contract. [Status](../status.md) indexes delivery and remaining acceptance. frame matches its desktop source dimensions. Other frames retain their own aspect ratio. This changes only the encoded presentation, never the game or display settings; vendor-private scaling is not inferred. + Each Windows output fits that presentation within its selected ceiling using + even dimensions, without enlarging the source or encoding a fixed landscape + canvas. The existing WebRTC encoder may adapt further; source resizing does + not create a new room or route. Frame envelopes carry actual output dimensions. - Windows SDR conversion owns one output color contract: limited-range BT.601 NV12 with matching H.264 metadata. RGB capture and decoded video retain their input range/matrix through conversion and relay scaling; dimensions do not diff --git a/docs/status.md b/docs/status.md index c99d58de8..b0158532a 100644 --- a/docs/status.md +++ b/docs/status.md @@ -65,7 +65,9 @@ Numeric audio is implemented in the unpublished candidate: Browser/server v24 and Native control v10 replace the preset field; capture v7 and SQLite schema 2 remain unchanged. Matched candidate App/Web/Server builds are required; 1.x participants are rejected by the existing compatibility checks. -Geometry, 4K and additional codecs are still assessed through the media boundary. +Windows output now preserves the source shape within the existing ceiling; +axis-independent limits, other-platform geometry, 4K and additional codecs are +still assessed through the media boundary. Native Magicsock remains an isolated experiment with incomplete product integration and unresolved recovery cases. These proposals do not yet change the accepted contract or transport. A small external test is proposed before diff --git a/docs/todo.md b/docs/todo.md index 68e04f187..b0a79896c 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -50,7 +50,10 @@ Implementation needs a concrete useful result within the product model. actual output. Numeric audio admission, storage and the compact custom input implement [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s use case, with local Browser/direct/relay and Browser/native SFU evidence. Next assess - source-aspect-preserving geometry and 4K through the same owners. Preserve + source-aspect-preserving geometry and 4K through the same owners. Windows + output now fits the source without a padded landscape canvas or upscaling; + axis-independent limits, Browser/other-platform geometry and product 4K + admission remain open. Preserve safety limits and framework-owned adaptation. The [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment) retain explicit receive-limit enforcement and the current 320 kbps sender diff --git a/docs/verification-status.md b/docs/verification-status.md index 379155b84..09bb444e3 100644 --- a/docs/verification-status.md +++ b/docs/verification-status.md @@ -11,7 +11,8 @@ implementation and obsolete package history. Windows App and Browser are the owner's primary acceptance targets. Current Go packages embed their Web assets and do not run a Node backend. The current -contract is Browser/server v23, Native control v9 and capture v7. +published contract is Browser/server v23, Native control v9 and capture v7; +[status](./status.md#current-work) owns unpublished candidate changes. Bounded current checks cover: diff --git a/native/capture/windows/capture_geometry.h b/native/capture/windows/capture_geometry.h index 3c1bff3db..90c6ed704 100644 --- a/native/capture/windows/capture_geometry.h +++ b/native/capture/windows/capture_geometry.h @@ -4,6 +4,7 @@ #include #include +#include namespace piik::capture { @@ -40,4 +41,18 @@ inline RECT FitFrameRect(SIZE content, SIZE output) { return {left, top, left + width, top + height}; } +// A quality profile is an output ceiling, not a canvas to pad or upscale into. +// Keep even dimensions for NV12; WebRTC can further adapt this bounded source. +inline SIZE BoundedFrameSize(SIZE content, SIZE ceiling) { + if (content.cx <= 0 || content.cy <= 0 || ceiling.cx < 2 || ceiling.cy < 2) + throw std::invalid_argument("Invalid capture presentation dimensions"); + const double scale = std::min({1.0, + static_cast(ceiling.cx) / content.cx, + static_cast(ceiling.cy) / content.cy}); + return { + std::max(2, static_cast(content.cx * scale) & ~1L), + std::max(2, static_cast(content.cy * scale) & ~1L), + }; +} + } // namespace piik::capture diff --git a/native/capture/windows/capture_geometry.test.cpp b/native/capture/windows/capture_geometry.test.cpp index 766d11ab8..c9b75a7ec 100644 --- a/native/capture/windows/capture_geometry.test.cpp +++ b/native/capture/windows/capture_geometry.test.cpp @@ -49,5 +49,19 @@ int main() { {240, 0, 1680, 1080}); const RECT wide = FitFrameRect({2560, 1080}, monitor); assert(wide.left == 0 && wide.right == 1920 && wide.top > 0 && wide.bottom < 1080); + const auto expect_size = [&](SIZE content, SIZE expected) { + const SIZE actual = BoundedFrameSize(content, monitor); + assert(actual.cx == expected.cx && actual.cy == expected.cy); + }; + expect_size({1080, 1920}, {606, 1080}); + expect_size({3440, 1440}, {1920, 802}); + expect_size({640, 480}, {640, 480}); + expect_size({641, 481}, {640, 480}); + expect_size({3840, 2160}, monitor); + expect_size(presentation(DISPLAYCONFIG_SCALING_STRETCHED), monitor); + try { + (void)BoundedFrameSize({}, monitor); + assert(false); + } catch (const std::invalid_argument&) {} std::cout << "Windows capture presentation geometry passed.\n"; } diff --git a/native/capture/windows/main.cpp b/native/capture/windows/main.cpp index 640a40c6c..804615e27 100644 --- a/native/capture/windows/main.cpp +++ b/native/capture/windows/main.cpp @@ -1471,6 +1471,8 @@ class OutputWorker final { } std::optional output; try { + const auto size = piik::capture::BoundedFrameSize(input->presentation, + {static_cast(profile_.width), static_cast(profile_.height)}); if (!encoder) { encoder = std::make_unique( kind_, profile_, device_.Get(), create_, std::move(initial), layer_); @@ -1486,7 +1488,8 @@ class OutputWorker final { } return converter->Convert(input->texture.Get(), input->width, input->height, input->presentation, input->color_space); - }, profile_.width, profile_.height, input->timestamp, work.recovery, work.bitrate); + }, static_cast(size.cx), static_cast(size.cy), + input->timestamp, work.recovery, work.bitrate); } catch (...) { // Device loss is current shared evidence; an old codec failure only // retires its activation. No retry occurs without a new activation. diff --git a/native/capture/windows/output_worker.test.cpp b/native/capture/windows/output_worker.test.cpp index 2c85e316d..1939f4d37 100644 --- a/native/capture/windows/output_worker.test.cpp +++ b/native/capture/windows/output_worker.test.cpp @@ -72,6 +72,7 @@ void CheckWorkerFailureGeneration(ID3D11Device* device, bool replace) { auto input = std::make_shared(); input->width = 1280; input->height = 720; + input->presentation = {1280, 720}; input->timestamp = 333'333; worker.Submit(input); assert(began.wait_for(std::chrono::seconds(2)) == std::future_status::ready); @@ -115,6 +116,36 @@ void CheckPrimaryFailureDetails() { assert(rejected); } +void CheckWorkerDimensions(ID3D11Device* device, SIZE content, SIZE expected) { + HANDLE read = nullptr, write = nullptr; + assert(CreatePipe(&read, &write, nullptr, 4096)); + UniqueHandle input_pipe(read), output_pipe(write); + ProtocolWriter writer(write); + std::promise output; + auto delivered = output.get_future(); + OutputWorker worker(0, VideoProfile{}, OutputKind::h264, device, writer, + [](const VideoProfile&) -> std::unique_ptr { + return std::make_unique(nullptr, std::shared_future{}); + }, nullptr, true, [&](const EncodedAccessUnit&) { output.set_value(); }, + [&](std::exception_ptr error) { output.set_exception(error); }); + auto frame = std::make_shared(); + frame->width = static_cast(content.cx); + frame->height = static_cast(content.cy); + frame->presentation = content; + frame->timestamp = 333'333; + worker.Submit(frame); + assert(delivered.wait_for(std::chrono::seconds(2)) == std::future_status::ready); + delivered.get(); + worker.Stop(); + worker.Join(); + std::array packet{}; + DWORD received = 0; + assert(ReadFile(read, packet.data(), static_cast(packet.size()), &received, nullptr)); + assert(received == packet.size()); + assert((packet[24] << 8 | packet[25]) == expected.cx); + assert((packet[26] << 8 | packet[27]) == expected.cy); +} + void CheckH264ProfileCompatibility() { const VideoProfile profile; for (const auto* value : {"42c01f", "42e01f", "42E01F", "4d801f", "58c01f"}) @@ -302,4 +333,7 @@ int main() { CheckAutoCadenceProbe(device.Get()); CheckWorkerFailureGeneration(device.Get(), false); CheckWorkerFailureGeneration(device.Get(), true); + CheckWorkerDimensions(device.Get(), {1080, 1920}, {404, 720}); + CheckWorkerDimensions(device.Get(), {3440, 1440}, {1280, 534}); + CheckWorkerDimensions(device.Get(), {640, 480}, {640, 480}); } diff --git a/scripts/app-media-gate.ts b/scripts/app-media-gate.ts index 67b803892..9bd1ca647 100644 --- a/scripts/app-media-gate.ts +++ b/scripts/app-media-gate.ts @@ -3,6 +3,7 @@ import { copyFile, mkdir, mkdtemp } from "node:fs/promises"; import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; +import { fittedCaptureSize, matchesCaptureSize } from "./media-gate-geometry"; import { CdpConnection, @@ -266,7 +267,7 @@ async function browserMediaGate(input: { sourceTitle: string; sourceKind: "window" | "display"; stunUrls: string; -}): Promise { +}, geometry: { fit: typeof fittedCaptureSize; matches: typeof matchesCaptureSize }): Promise { let socket: WebSocket | null = null; const peers: RTCPeerConnection[] = []; const result: MediaEvidence = { @@ -436,10 +437,10 @@ async function browserMediaGate(input: { encoderIndex: encoder.index, edgeCapacity: 2, profile: { - resolution: "720p", + resolution: "1440p", maxFramerate: 30, maxBitrate: 3_000_000, - degradationPreference: "balanced", + degradationPreference: "maintain-resolution", maxAudioBitrate: 128000, }, }); @@ -495,6 +496,11 @@ async function browserMediaGate(input: { await peer.addIceCandidate(candidate); } const answer = await peer.createAnswer(); + // This fixture is an unchanged-Opus receiver, like the product Viewer. + // Declare its receive capacity; the offer's fmtp is the opposite direction. + const opus = answer.sdp?.match(/^a=rtpmap:(\d+) opus\/48000\/2\r?$/m)?.[1]; + if (opus) answer.sdp = answer.sdp!.replace(new RegExp(`^(a=fmtp:${opus} [^\\r\\n]*)`, "m"), + "$1;stereo=1;maxaveragebitrate=510000"); await peer.setLocalDescription(answer); await request("edge-answer", { shareId, @@ -552,16 +558,16 @@ async function browserMediaGate(input: { ? Math.min(...result.edgeAudioBytes) : 0; const videos = [...edges.values()].map((edge) => edge.video); - result.width = videos.every((video) => video.videoWidth === 1280) ? 1280 : 0; - result.height = videos.every((video) => video.videoHeight === 720) ? 720 : 0; + result.width = videos.every((video) => video.videoWidth === videos[0]!.videoWidth && video.videoWidth <= 2560) ? videos[0]!.videoWidth : 0; + result.height = videos.every((video) => video.videoHeight === videos[0]!.videoHeight && video.videoHeight <= 1440) ? videos[0]!.videoHeight : 0; if ( result.captureActive && result.connected && result.frames >= 30 && (!result.audioAvailable || (result.audioPackets > 10 && result.audioEnergy > 0)) && - result.width === 1280 && - result.height === 720 && + result.width >= 320 && + result.height >= 240 && result.nativeCandidateTypes.includes("srflx") ) { break; @@ -574,21 +580,24 @@ async function browserMediaGate(input: { result.frames < 30 || (result.audioAvailable && (result.audioPackets <= 10 || result.audioEnergy <= 0)) || - result.width !== 1280 || - result.height !== 720 || + result.width < 320 || + result.height < 240 || !result.nativeCandidateTypes.includes("srflx") ) { throw new Error("Native H264 did not reach decoded Browser video"); } const videos = [...edges.values()].map((edge) => edge.video); + const sourceSize = { width: result.width, height: result.height }; + const liveSize = geometry.fit(sourceSize, 1280, 720); + const pausedSize = geometry.fit(sourceSize, 854, 480); const framesBeforeLiveUpdate = Math.min( ...videos.map((video) => video.getVideoPlaybackQuality().totalVideoFrames), ); const liveUpdate = await request("update-share", { shareId, profile: { - resolution: "1440p", + resolution: "720p", maxFramerate: 60, maxBitrate: 12_000_000, degradationPreference: "maintain-framerate", @@ -601,7 +610,7 @@ async function browserMediaGate(input: { ...videos.map((video) => video.getVideoPlaybackQuality().totalVideoFrames), ); result.liveProfileUpdated = liveUpdate.type === "share-updated" && - videos.every((video) => video.videoWidth === 2560 && video.videoHeight === 1440) && + videos.every((video) => geometry.matches(video.videoWidth, video.videoHeight, liveSize)) && frames >= framesBeforeLiveUpdate + 10; if (result.liveProfileUpdated) break; await new Promise((resolveWait) => window.setTimeout(resolveWait, 100)); @@ -631,12 +640,12 @@ async function browserMediaGate(input: { ...videos.map((video) => video.getVideoPlaybackQuality().totalVideoFrames), ); result.pausedProfileUpdated = pausedUpdate.type === "share-updated" && - videos.every((video) => video.videoWidth === 854 && video.videoHeight === 480) && + videos.every((video) => geometry.matches(video.videoWidth, video.videoHeight, pausedSize)) && frames >= framesBeforeResume + 10; if (result.pausedProfileUpdated) { result.frames = frames; - result.width = 854; - result.height = 480; + result.width = videos[0]!.videoWidth; + result.height = videos[0]!.videoHeight; break; } await new Promise((resolveWait) => window.setTimeout(resolveWait, 100)); @@ -901,7 +910,7 @@ async function main(): Promise { sourceTitle: SOURCE_TITLE, sourceKind, stunUrls: GATE_STUN_URLS, - }) + "))((target) => target)", + }) + `, { fit: ${fittedCaptureSize.toString()}, matches: ${matchesCaptureSize.toString()} }))((target) => target)`, Date.now() + 70_000, ); if (media.error) throw new Error(media.error); @@ -925,7 +934,7 @@ async function main(): Promise { (!media.audioAvailable || (media.audioPackets > 10 && media.audioEnergy > 0)) && media.liveProfileUpdated && media.pausedProfileUpdated && - media.width === 854 && media.height === 480 && + media.width > 0 && media.width <= 854 && media.height > 0 && media.height <= 480 && media.nativeCandidateTypes.includes("srflx") && cleanup.browserExited && cleanup.nativeExited && cleanup.serverClosed && cleanup.portsClosed && cleanup.profileRemoved; diff --git a/scripts/app-native-host-gate.ts b/scripts/app-native-host-gate.ts index 9683c6975..494a7ab15 100644 --- a/scripts/app-native-host-gate.ts +++ b/scripts/app-native-host-gate.ts @@ -10,6 +10,7 @@ import { mkdir, mkdtemp, readFile, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; +import { fittedCaptureSize, matchesCaptureSize } from "./media-gate-geometry"; import { CdpConnection, @@ -35,6 +36,11 @@ type GateMode = "local" | "one-link"; type VideoCodec = "h264" | "vp8"; const CODEC_PROBE = `(() => { + const fitSize = ${fittedCaptureSize.toString()}, matchesSize = ${matchesCaptureSize.toString()}; + window.__piikGateFitsProfile = (width, height) => { + const video = document.querySelector('video'), source = window.__piikGateSourceSize; + return Boolean(video && source && matchesSize(video.videoWidth, video.videoHeight, fitSize(source, width, height))); + }; const peers = [], BrowserPeer = RTCPeerConnection; const sockets = [], BrowserSocket = WebSocket; window.WebSocket = class extends BrowserSocket { @@ -783,6 +789,10 @@ async function main(): Promise { 5_000, ); stage = "pre-share-codec"; + // Observe the source at the largest current ceiling before testing smaller + // ceilings. Native capture no longer pads/upscales every source to 16:9. + await clickHostControl(cdp, host, `document.querySelector('button.lr-chip[aria-label="1440p"]')`); + await clickHostControl(cdp, host, `document.querySelector('button[name="degradationPreference"][value="maintain-resolution"]')`); await clickHostControl(cdp, host, `document.querySelector('.lr-sharing-technical > summary')`); await clickHostControl(cdp, host, `([...document.querySelectorAll('button.lr-chip')].find((button) => @@ -906,13 +916,16 @@ async function main(): Promise { deadline, ), (state) => state.connected && state.frames >= 30 && - state.width === 1920 && state.height === 1080, + state.width >= 320 && state.width <= 2560 && state.height >= 240 && state.height <= 1440, 40_000, ); result.viewerConnected = viewerState.connected; result.viewerFrames = viewerState.frames; result.viewerWidth = viewerState.width; result.viewerHeight = viewerState.height; + await evaluate(cdp, viewer, + `window.__piikGateSourceSize = ${JSON.stringify({ width: viewerState.width, height: viewerState.height })}`, + Date.now() + 5_000); await assertVideoCodec(cdp, viewer, actualCodec); if (mode === "one-link") { result.codecPreserved = true; @@ -943,9 +956,7 @@ async function main(): Promise { (deadline) => evaluate( cdp!, viewer, - `(() => { const video = document.querySelector('video'); return Boolean( - video && video.videoWidth === ${width} && video.videoHeight === ${height} - ); })()`, + `window.__piikGateFitsProfile(${width}, ${height})`, deadline, ), Boolean, @@ -993,7 +1004,7 @@ async function main(): Promise { cdp!, viewer, `(() => { const video = document.querySelector('video'); return Boolean( - video && video.videoWidth === 2560 && video.videoHeight === 1440 && + video && window.__piikGateFitsProfile(2560, 1440) && video.getVideoPlaybackQuality().totalVideoFrames >= ${viewerState.frames + 10} ); })()`, deadline, @@ -1004,8 +1015,8 @@ async function main(): Promise { if (!result.liveQualityChanged) { throw new Error("Native quality change did not reach the Viewer"); } - result.viewerWidth = 2560; - result.viewerHeight = 1440; + result.viewerWidth = viewerState.width; + result.viewerHeight = viewerState.height; await assertVideoCodec(cdp, viewer, actualCodec); const framesBeforePreference = await evaluate( cdp, @@ -1132,10 +1143,10 @@ async function main(): Promise { deadline, ), (value) => value.frames >= framesBeforeResume + 10 && - value.width === 854 && value.height === 480, + matchesCaptureSize(value.width, value.height, fittedCaptureSize(viewerState, 854, 480)), 20_000, ); - result.pausedQualityChanged = resumed.width === 854 && resumed.height === 480; + result.pausedQualityChanged = matchesCaptureSize(resumed.width, resumed.height, fittedCaptureSize(viewerState, 854, 480)); result.mediaObjectPreserved = resumed.sameMedia; result.viewerWidth = resumed.width; result.viewerHeight = resumed.height; @@ -1189,7 +1200,7 @@ async function main(): Promise { `(() => { const video = document.querySelector('video'); return Boolean(video && video.srcObject === window.__piikGateMedia && - video.videoWidth === ${width} && video.videoHeight === ${height} && + window.__piikGateFitsProfile(${width}, ${height}) && video.getVideoPlaybackQuality().totalVideoFrames >= ${before + 10}); })()`, deadline, ), Boolean, 20_000); @@ -1278,7 +1289,7 @@ async function main(): Promise { const video = document.querySelector('video'); return Boolean( video && video.srcObject === window.__piikGateMedia && - video.videoWidth === 854 && video.videoHeight === 480 && + video.videoWidth > 0 && video.videoWidth <= 854 && video.videoHeight > 0 && video.videoHeight <= 480 && video.getVideoPlaybackQuality().totalVideoFrames >= ${framesBeforeSourceChange + 10} ); })()`, @@ -1386,7 +1397,7 @@ async function main(): Promise { const video = document.querySelector('video'); return Boolean(video && video.srcObject && video.srcObject !== window.__piikGateMedia && - video.videoWidth === 854 && video.videoHeight === 480); + video.videoWidth > 0 && video.videoWidth <= 854 && video.videoHeight > 0 && video.videoHeight <= 480); })()`, deadline, ), @@ -1460,7 +1471,8 @@ async function main(): Promise { result.remoteNatPath && result.remotePeerExited === true : mode === "one-link" ? result.publicViewerPage && result.publicViewerSignal && result.viewerConnected && - result.viewerFrames >= 30 && result.viewerWidth === 1920 && result.viewerHeight === 1080 && result.codecPreserved + result.viewerFrames >= 30 && result.viewerWidth > 0 && result.viewerWidth <= 2560 && + result.viewerHeight > 0 && result.viewerHeight <= 1440 && result.codecPreserved : result.viewerConnected && result.viewerFrames >= 30 && result.qualityControlsEnabled && result.liveQualityChanged && result.backgroundProfileRecovery !== false && @@ -1469,7 +1481,7 @@ async function main(): Promise { result.codecPreserved && result.pausedQualityChanged && result.mediaObjectPreserved && result.nativeSourceChanged && - result.viewerWidth === 854 && result.viewerHeight === 480 && + result.viewerWidth > 0 && result.viewerWidth <= 854 && result.viewerHeight > 0 && result.viewerHeight <= 480 && result.nativeQualityEvidence && (crashGate ? result.appCrashEndedShare === true diff --git a/scripts/embedded-sfu-gate.ts b/scripts/embedded-sfu-gate.ts index a6d0fbc78..2da845877 100644 --- a/scripts/embedded-sfu-gate.ts +++ b/scripts/embedded-sfu-gate.ts @@ -12,6 +12,7 @@ import { import type { snapshot } from "./embedded-sfu-page"; import { readAppEndpoint } from "./app-gate-endpoint"; import { SOURCE_TITLE, sourceServer, startSourceBrowser, waitForCaptureWindow } from "./app-native-host-gate"; +import { fittedCaptureSize } from "./media-gate-geometry"; const ROOT = resolve(import.meta.dirname, ".."); const BUILD_ROOT = join(ROOT, "build", "embedded-media"); @@ -251,7 +252,7 @@ async function main(): Promise { await until(secondHost, (value) => value.authenticated); await call(secondViewer, `gate.startViewer(${JSON.stringify(secondRoom)})`); await until(secondViewer, (value) => value.committed && value.audioKbps > 0); - await call(secondViewer, "gate.waitForFrames(1280, 720)"); + await call(secondViewer, "gate.waitForFrames(1280, 720, 15, true)"); await waitForSample(() => call(secondViewer!, "gate.audioEnergy()", 3_000), (energy) => { result.secondRoom!.audioBefore = energy; return energy > 0; }, 5_000); const second = await call(secondHost, "gate.snapshot()"); @@ -266,7 +267,7 @@ async function main(): Promise { : null; const sourceBefore = await sourceSample(); try { - result.high = await call(viewer, nativeArm ? "gate.waitForFrames(1280, 720, 300)" : "gate.waitForFrames(1280, 720)", nativeArm ? 32_000 : 25_000); + result.high = await call(viewer, nativeArm ? "gate.waitForFrames(1280, 720, 300, true)" : "gate.waitForFrames(1280, 720)", nativeArm ? 32_000 : 25_000); } finally { const sourceAfter = await sourceSample().catch(() => null); if (sourceBefore && sourceAfter) result.sourceDrawing = { @@ -285,7 +286,7 @@ async function main(): Promise { let previousEnergy = result.audioEnergy.high; for (const bitrate of [320_000, 96_000]) { if (!await call(host, `gate.setAudioBitrate(${bitrate})`)) throw new Error("Numeric audio update failed"); - const frames = await call<{ frames: number }>(viewer, "gate.waitForFrames(1280, 720)"); + const frames = await call<{ frames: number }>(viewer, `gate.waitForFrames(${result.high!.width}, ${result.high!.height})`); const after = await call(viewer, "gate.snapshot()"); const energy = await call(viewer, "gate.audioEnergy()", 3_000); const retained = after.connectionId === audioConnection && after.committed; @@ -326,7 +327,8 @@ async function main(): Promise { result.host = await call(host, "gate.snapshot()"); throw new Error("Live profile update failed"); } - result.low = await call(viewer, "gate.waitForFrames(854, 480)"); + const lowSize = nativeArm ? fittedCaptureSize(result.high!, 854, 480) : { width: 854, height: 480 }; + result.low = await call(viewer, `gate.waitForFrames(${lowSize.width}, ${lowSize.height}, 15, false, ${nativeArm ? 2 : 0})`); await waitForSample(() => call(viewer!, "gate.audioEnergy()", 3_000), (energy) => { result.audioEnergy.low = energy; return energy > result.audioEnergy.high; }, 5_000); result.stage = "viewer-media-reload"; @@ -335,7 +337,7 @@ async function main(): Promise { // still retains the old committed edge inside the reconnect grace period. await call(viewer, `gate.reconnectViewer(${JSON.stringify(room)}, true)`); await until(viewer, value => value.committed && value.connectionId !== retainedSubscription && value.audioKbps > 0); - await call(viewer, "gate.waitForFrames(854, 480)"); + await call(viewer, `gate.waitForFrames(${result.low!.width}, ${result.low!.height})`); result.pageReloadRecovered = true; result.stage = "committed-subscription-recovery"; const previousSubscription = await call(viewer, "gate.failSubscription()"); @@ -344,7 +346,7 @@ async function main(): Promise { await until(viewer, value => value.routeStatus === "failed"); await call(viewer, `gate.reconnectViewer(${JSON.stringify(room)})`); await until(viewer, value => value.committed && value.connectionId !== previousSubscription && value.audioKbps > 0); - await call(viewer, "gate.waitForFrames(854, 480)"); + await call(viewer, `gate.waitForFrames(${result.low!.width}, ${result.low!.height})`); await waitForSample(() => call(viewer!, "gate.audioEnergy()", 3_000), energy => { result.audioEnergy.recovered = energy; return energy > 0; }, 5_000); result.viewer = await call(viewer, "gate.snapshot()"); @@ -360,7 +362,7 @@ async function main(): Promise { if (!(await call(host, "gate.snapshot()")).nativeShareStopped) throw new Error("First Native capture did not stop"); result.secondRoom!.audioBefore = await call(secondViewer, "gate.audioEnergy()"); await cdp.call("Page.bringToFront", {}, secondViewer.sessionId, Date.now() + 3_000); - const continued = await call<{ frames: number }>(secondViewer, "gate.waitForFrames(1280, 720, 30)"); + const continued = await call<{ frames: number }>(secondViewer, `gate.waitForFrames(${result.high!.width}, ${result.high!.height}, 30)`); result.secondRoom!.framesAfterFirstStopped = continued.frames; await waitForSample(() => call(secondViewer!, "gate.audioEnergy()", 3_000), (energy) => { result.secondRoom!.audioAfter = energy; return energy > result.secondRoom!.audioBefore; }, 5_000); diff --git a/scripts/embedded-sfu-page.ts b/scripts/embedded-sfu-page.ts index dba5c67c7..1efa47f29 100644 --- a/scripts/embedded-sfu-page.ts +++ b/scripts/embedded-sfu-page.ts @@ -350,7 +350,7 @@ export async function setAudioBitrate(maxAudioBitrate: number): Promise return true; } -export async function waitForFrames(width: number, height: number, minimumFrames = 15) { +export async function waitForFrames(width: number, height: number, minimumFrames = 15, withinCeiling = false, tolerance = 0) { if (!video) throw new Error("Viewer video is unavailable"); const target = video; return await new Promise<{ frames: number; width: number; height: number }>((resolve, reject) => { @@ -362,7 +362,10 @@ export async function waitForFrames(width: number, height: number, minimumFrames }, minimumFrames > 15 ? 30_000 : 20_000); const next = () => { callback = target.requestVideoFrameCallback(() => { - frames = target.videoWidth === width && target.videoHeight === height ? frames + 1 : 0; + const matches = withinCeiling + ? target.videoWidth >= 320 && target.videoHeight >= 240 && target.videoWidth <= width && target.videoHeight <= height + : Math.abs(target.videoWidth - width) <= tolerance && Math.abs(target.videoHeight - height) <= tolerance; + frames = matches ? frames + 1 : 0; if (frames < minimumFrames) return next(); clearTimeout(timer); resolve({ frames, width: target.videoWidth, height: target.videoHeight }); diff --git a/scripts/media-gate-geometry.ts b/scripts/media-gate-geometry.ts new file mode 100644 index 000000000..1d388ee5e --- /dev/null +++ b/scripts/media-gate-geometry.ts @@ -0,0 +1,11 @@ +// Gate expectations derive from the decoded source instead of assuming a 16:9 +// canvas. Two pixels allow independent NV12 rounding at successive ceilings. +export function fittedCaptureSize(source: { width: number; height: number }, width: number, height: number) { + const scale = Math.min(1, width / source.width, height / source.height); + return { width: Math.max(2, Math.floor(source.width * scale / 2) * 2), + height: Math.max(2, Math.floor(source.height * scale / 2) * 2) }; +} + +export function matchesCaptureSize(width: number, height: number, expected: { width: number; height: number }) { + return width > 0 && height > 0 && Math.abs(width - expected.width) <= 2 && Math.abs(height - expected.height) <= 2; +} From 8b46d7c59930bba7709f84997d5dcb67c3893f2f Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:12:06 +0800 Subject: [PATCH 069/262] test(media): declare Opus limits in native and SFU receive fixtures --- internal/app/nativehost/audio_test.go | 13 +++++++++++++ internal/server/signal/router_sfu_signal_test.go | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/internal/app/nativehost/audio_test.go b/internal/app/nativehost/audio_test.go index f2fc435ee..51bd8725a 100644 --- a/internal/app/nativehost/audio_test.go +++ b/internal/app/nativehost/audio_test.go @@ -150,6 +150,19 @@ func TestAudioEOFWaitsForExplicitSourceReplacement(t *testing.T) { offer, err := session.PrepareLocalEdge("audio-edge") check(err) check(receiver.SetRemoteDescription(offer)) + // A receive preference belongs to this answer, not the upstream offer. + for _, transceiver := range receiver.GetTransceivers() { + if transceiver.Kind() != webrtc.RTPCodecTypeAudio { + continue + } + for _, codec := range transceiver.Receiver().GetParameters().Codecs { + if codec.MimeType == webrtc.MimeTypeOpus { + codec.SDPFmtpLine = "stereo=1;maxaveragebitrate=510000" + check(transceiver.SetCodecPreferences([]webrtc.RTPCodecParameters{codec})) + break + } + } + } go func() { for { select { diff --git a/internal/server/signal/router_sfu_signal_test.go b/internal/server/signal/router_sfu_signal_test.go index f8068c576..e5e92f4bc 100644 --- a/internal/server/signal/router_sfu_signal_test.go +++ b/internal/server/signal/router_sfu_signal_test.go @@ -210,6 +210,20 @@ func TestEmbeddedSfuAuthenticatedWebSocketCarriesActualRTP(t *testing.T) { } }) applyRemote(viewer, subscriber, viewerConfig, false) + // SFU reserves its audio track for later source/microphone changes. Match + // the product Viewer's receive preference even while this fixture is silent. + for _, transceiver := range subscriber.GetTransceivers() { + if transceiver.Kind() != webrtc.RTPCodecTypeAudio { + continue + } + for _, codec := range transceiver.Receiver().GetParameters().Codecs { + if codec.MimeType == webrtc.MimeTypeOpus { + codec.SDPFmtpLine = "stereo=1;maxaveragebitrate=510000" + check(transceiver.SetCodecPreferences([]webrtc.RTPCodecParameters{codec})) + break + } + } + } answer, err := subscriber.CreateAnswer(nil) check(err) sendDescription(viewer, viewerConfig, gather(subscriber, answer), nil) From 97f4066c9fe2c5597ea8f91fd379121606f24508 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:23:25 +0800 Subject: [PATCH 070/262] refactor(capture): validate numeric profiles independently of UI presets --- docs/research/native-client-media.md | 14 +++++++------ internal/app/nativecapture/protocol_test.go | 16 ++++++++++++++- internal/app/nativecapture/stream.go | 10 ++++------ native/capture/darwin/main.swift | 20 +++++++++---------- native/capture/linux/main.c | 18 ++++++++--------- native/capture/linux/main.test.c | 18 +++++++++++++++++ native/capture/windows/capture_geometry.h | 5 +++++ native/capture/windows/main.cpp | 11 +++------- native/capture/windows/output_worker.test.cpp | 17 ++++++++++------ 9 files changed, 83 insertions(+), 46 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 82508f7c0..441259ecb 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -556,12 +556,14 @@ changes, Viewer reload/recovery and cleanup. Gate expectations now derive from the observed source and selected ceilings, instead of requiring encoded 16:9 padding. Physical game, macOS/Linux and orientation-change acceptance remain open. -The restriction extends beyond resolution labels: shared readers also bound -frame rate and bitrate, and Native's selectable profile admits four exact raster -pairs while derived outputs admit other dimensions within axis-specific limits. -These are different meanings. Keep explicit resource ceilings, but assess them -separately from presets and device capability; arbitrary numbers are not the -replacement contract. +The candidate's Native capture admission now shares the numeric dimension +bounds already used by encoded outputs instead of duplicating the four UI +presets. Even dimensions must fit within 2560x1440; a source ceiling must also +yield a valid half-size output. Frame-rate, bitrate and output-count bounds +remain unchanged. Signaling still uses the existing resolution enum: custom +dimensions are an internal foundation, not yet a product control or a 4K option. +Device capability and the public request contract still need separate admission; +removing an enum is not permission to allocate arbitrary media resources. The proposed boundary follows the [Media Capture specification](https://www.w3.org/TR/mediacapture-streams/#constrainable-interface): diff --git a/internal/app/nativecapture/protocol_test.go b/internal/app/nativecapture/protocol_test.go index d38628432..b356ada63 100644 --- a/internal/app/nativecapture/protocol_test.go +++ b/internal/app/nativecapture/protocol_test.go @@ -290,13 +290,27 @@ func TestVideoProfileUsesTheProductBounds(t *testing.T) { {Width: 1280, Height: 720, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, {Width: 1920, Height: 1080, Framerate: 60, Bitrate: 8_000_000, Preference: "maintain-framerate"}, {Width: 2560, Height: 1440, Framerate: 60, Bitrate: 12_000_000, Preference: "balanced"}, + {Width: 1920, Height: 1200, Framerate: 30, Bitrate: 5_000_000, Preference: "balanced"}, + {Width: 720, Height: 1280, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 4, Height: 4, Framerate: 15, Bitrate: 2_000_000, Preference: "balanced"}, } { if !profile.Valid() { t.Fatalf("valid profile rejected: %+v", profile) } + for _, output := range ScreenShareOutputs(profile) { + if !output.Valid() { + t.Fatalf("valid source profile produced an invalid output: %+v", output) + } + } } for _, profile := range []VideoProfile{ - {Width: 1920, Height: 1200, Framerate: 30, Bitrate: 5_000_000, Preference: "balanced"}, + {Width: 0, Height: 720, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 2, Height: 720, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 1280, Height: 2, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 1279, Height: 720, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 1280, Height: 719, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 2562, Height: 1440, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 2560, Height: 1442, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, {Width: 1920, Height: 1080, Framerate: 14, Bitrate: 5_000_000, Preference: "balanced"}, {Width: 1920, Height: 1080, Framerate: 30, Bitrate: 12_000_001, Preference: "balanced"}, {Width: 1920, Height: 1080, Framerate: 30, Bitrate: 5_000_000, Preference: "unknown"}, diff --git a/internal/app/nativecapture/stream.go b/internal/app/nativecapture/stream.go index 0df559bb3..205eb63d9 100644 --- a/internal/app/nativecapture/stream.go +++ b/internal/app/nativecapture/stream.go @@ -109,12 +109,10 @@ func ScreenShareOutputs(profile VideoProfile) []OutputProfile { } func (profile VideoProfile) Valid() bool { - validResolution := - (profile.Width == 854 && profile.Height == 480) || - (profile.Width == 1280 && profile.Height == 720) || - (profile.Width == 1920 && profile.Height == 1080) || - (profile.Width == 2560 && profile.Height == 1440) - return validResolution && profile.Framerate >= 15 && profile.Framerate <= 60 && + // The source ceiling must also yield a valid half-size output. Preset labels + // belong to the UI, not the capture process's numeric resource boundary. + return ValidVideoSize(profile.Width, profile.Height) && profile.Width >= 4 && profile.Height >= 4 && + profile.Framerate >= 15 && profile.Framerate <= 60 && profile.Bitrate >= 2_000_000 && profile.Bitrate <= 12_000_000 && validVideoPreference(profile.Preference) } diff --git a/native/capture/darwin/main.swift b/native/capture/darwin/main.swift index fc4986a7c..05a72adbb 100644 --- a/native/capture/darwin/main.swift +++ b/native/capture/darwin/main.swift @@ -1340,6 +1340,11 @@ private func captureFilter( throw CaptureFailure(description: "capture target kind is unsupported") } +private func validVideoSize(_ width: Int, _ height: Int) -> Bool { + width >= 2 && width <= 2560 && width % 2 == 0 && + height >= 2 && height <= 1440 && height % 2 == 0 +} + private func videoProfile(_ arguments: [String]) throws -> VideoProfile { guard arguments.count >= 28, arguments[10] == "--width", let width = Int(arguments[11]), @@ -1353,12 +1358,8 @@ private func videoProfile(_ arguments: [String]) throws -> VideoProfile { arguments[22] == "--protocol-v7" else { throw CaptureFailure(description: "invalid video profile arguments") } - let validResolution = - (width == 854 && height == 480) || - (width == 1280 && height == 720) || - (width == 1920 && height == 1080) || - (width == 2560 && height == 1440) - guard validResolution, frameRate >= 15, frameRate <= 60, + guard validVideoSize(width, height), width >= 4, height >= 4, + frameRate >= 15, frameRate <= 60, bitrate >= 2_000_000, bitrate <= 12_000_000 else { throw CaptureFailure(description: "video profile is outside the product bounds") } @@ -1384,8 +1385,8 @@ private func outputProfiles(_ arguments: [String], start: Int = 23, let height = Int(arguments[index + 2]), let fps = Int32(arguments[index + 3]), let bitrate = Int(arguments[index + 4]), - width >= 2, width <= (source?.width ?? 2560), width % 2 == 0, - height >= 2, height <= (source?.height ?? 1440), height % 2 == 0, + validVideoSize(width, height), + width <= (source?.width ?? 2560), height <= (source?.height ?? 1440), fps >= 1, fps <= (source?.frameRate ?? 60), bitrate >= 1000, bitrate <= (source?.bitrate ?? 12_000_000) else { throw CaptureFailure(description: "output profile is outside source bounds") @@ -1438,8 +1439,7 @@ private func readInput(done: StopSignal, outputs: [OutputProfile] = [], let timestamp = number(8..<16), duration = number(16..<24) let width = number(24..<26), height = number(26..<28) guard size <= maxPayloadBytes, header[6] <= 1, header[7] == 0, - width >= 2, width <= 2560, width % 2 == 0, - height >= 2, height <= 1440, height % 2 == 0, + validVideoSize(Int(width), Int(height)), timestamp <= UInt64(Int64.max) / 100, duration > 0, duration <= UInt64(Int64.max) / 100, let payload = try readExactly(size) else { diff --git a/native/capture/linux/main.c b/native/capture/linux/main.c index dd6c75427..5315881e5 100644 --- a/native/capture/linux/main.c +++ b/native/capture/linux/main.c @@ -370,6 +370,11 @@ static gboolean parse_uint(const char *value, guint minimum, guint maximum, return TRUE; } +static gboolean valid_video_size(guint width, guint height) { + return width >= 2 && width <= 2560 && width % 2 == 0 && + height >= 2 && height <= 1440 && height % 2 == 0; +} + static gboolean parse_outputs(int count, char **values, guint start, CaptureRun *run) { if (count <= (int)start || (count - (int)start) % 5 != 0 || (count - (int)start) / 5 > kMaxOutputs) return FALSE; @@ -389,7 +394,7 @@ static gboolean parse_outputs(int count, char **values, guint start, CaptureRun !parse_uint(values[offset + 2], 2, 1440, &output->profile.height) || !parse_uint(values[offset + 3], 1, 60, &output->profile.frame_rate) || !parse_uint(values[offset + 4], 1000, 12000000, &output->profile.bit_rate) || - output->profile.width % 2 != 0 || output->profile.height % 2 != 0) return FALSE; + !valid_video_size(output->profile.width, output->profile.height)) return FALSE; } return TRUE; } @@ -443,12 +448,8 @@ static gboolean parse_profile(int count, char **values, CaptureRun *run, !parse_uint(values[17], 2000000, 12000000, &profile->bit_rate)) { return FALSE; } - gboolean resolution = - (profile->width == 854 && profile->height == 480) || - (profile->width == 1280 && profile->height == 720) || - (profile->width == 1920 && profile->height == 1080) || - (profile->width == 2560 && profile->height == 1440); - if (!resolution) return FALSE; + if (!valid_video_size(profile->width, profile->height) || + profile->width < 4 || profile->height < 4) return FALSE; if (!parse_preference(values[19], profile) || !parse_outputs(count, values, 23, run)) return FALSE; run->original_output = MIN(1, run->output_count - 1); const VideoProfile *original = &run->outputs[run->original_output].profile; @@ -747,8 +748,7 @@ static gboolean valid_input_header(CaptureRun *run) { run->input_size == 0 || run->input_size > kMaxPayloadBytes) return FALSE; guint width = (guint)read_be(header + 24, 2), height = (guint)read_be(header + 26, 2); guint64 timestamp = read_be(header + 8, 8), duration = read_be(header + 16, 8); - return width >= 2 && width <= 2560 && height >= 2 && height <= 1440 && - width % 2 == 0 && height % 2 == 0 && duration > 0 && + return valid_video_size(width, height) && duration > 0 && timestamp <= INT64_MAX / 100 && duration <= INT64_MAX / 100; } diff --git a/native/capture/linux/main.test.c b/native/capture/linux/main.test.c index 142a04748..1052c9b82 100644 --- a/native/capture/linux/main.test.c +++ b/native/capture/linux/main.test.c @@ -152,6 +152,24 @@ static void check_slot_profiles(void) { for (guint index = 0; index < run.output_count; ++index) { g_assert_false(run.outputs[index].enabled); } + + char *numeric[] = { + "capture", "--capture-video", "picker", "1", "0", "0", + "--adapter-index", "0", "--mft-index", "0", "--width", "1920", + "--height", "1200", "--fps", "30", "--bitrate", "3000000", + "--preference", "balanced", "--codec", "h264", "--protocol-v7", + "--output", "1920", "1200", "30", "3000000"}; + const struct { char *width; char *height; gboolean valid; } sizes[] = { + {"1920", "1200", TRUE}, {"720", "1280", TRUE}, {"4", "4", TRUE}, + {"0", "720", FALSE}, {"2", "720", FALSE}, {"1280", "2", FALSE}, + {"1279", "720", FALSE}, {"1280", "719", FALSE}, + {"2562", "1440", FALSE}, {"2560", "1442", FALSE}, {"3840", "2160", FALSE}}; + for (guint index = 0; index < G_N_ELEMENTS(sizes); ++index) { + numeric[11] = numeric[24] = sizes[index].width; + numeric[13] = numeric[25] = sizes[index].height; + run = (CaptureRun){0}; + g_assert_cmpint(parse_profile(G_N_ELEMENTS(numeric), numeric, &run, &encoder), ==, sizes[index].valid); + } } static GstPadProbeReturn record_key_request(GstPad *pad, GstPadProbeInfo *info, diff --git a/native/capture/windows/capture_geometry.h b/native/capture/windows/capture_geometry.h index 90c6ed704..d73729873 100644 --- a/native/capture/windows/capture_geometry.h +++ b/native/capture/windows/capture_geometry.h @@ -8,6 +8,11 @@ namespace piik::capture { +inline bool ValidVideoSize(UINT32 width, UINT32 height) { + return width >= 2 && width <= 2560 && !(width & 1) && + height >= 2 && height <= 1440 && !(height & 1); +} + inline SIZE DisplayedFrameSize(SIZE captured, SIZE desktop, SIZE target, DISPLAYCONFIG_SCALING scaling, DISPLAYCONFIG_ROTATION rotation) { diff --git a/native/capture/windows/main.cpp b/native/capture/windows/main.cpp index 804615e27..b80b1261e 100644 --- a/native/capture/windows/main.cpp +++ b/native/capture/windows/main.cpp @@ -966,12 +966,8 @@ DegradationPreference ParseDegradationPreference(const wchar_t* value) { } void ValidateVideoProfile(const VideoProfile& profile) { - const bool valid_resolution = - (profile.width == 854 && profile.height == 480) || - (profile.width == 1280 && profile.height == 720) || - (profile.width == 1920 && profile.height == 1080) || - (profile.width == 2560 && profile.height == 1440); - if (!valid_resolution || profile.frame_rate < 15 || profile.frame_rate > 60 || + if (!piik::capture::ValidVideoSize(profile.width, profile.height) || + profile.width < 4 || profile.height < 4 || profile.frame_rate < 15 || profile.frame_rate > 60 || profile.bit_rate < 2'000'000 || profile.bit_rate > 12'000'000) { Fail("argument-profile", "video profile is outside the product bounds"); } @@ -985,8 +981,7 @@ void ParseOutputProfiles(ProductArguments& arguments, int first, int count, wcha output.height = ParseIndex(values[index + 2], "argument-output-height"); output.frame_rate = ParseIndex(values[index + 3], "argument-output-fps"); output.bit_rate = ParseIndex(values[index + 4], "argument-output-bitrate"); - if (output.width < 2 || output.height < 2 || (output.width & 1) || (output.height & 1) || - output.width > 2560 || output.height > 1440 || output.frame_rate == 0 || + if (!piik::capture::ValidVideoSize(output.width, output.height) || output.frame_rate == 0 || output.frame_rate > 60 || output.bit_rate < 1'000 || output.bit_rate > 12'000'000) { Fail("argument-output", "output profile exceeds codec bounds"); } diff --git a/native/capture/windows/output_worker.test.cpp b/native/capture/windows/output_worker.test.cpp index 1939f4d37..f74a6cc4d 100644 --- a/native/capture/windows/output_worker.test.cpp +++ b/native/capture/windows/output_worker.test.cpp @@ -315,12 +315,17 @@ int main() { assert(profile.h264_level() == 51 && profile.profile_level_id() == "42c033"); profile = VideoProfile{1920, 1080, 60, 8'000'000}; assert(profile.h264_level() == 42); - profile = VideoProfile{3840, 2160, 30, 12'000'000}; - try { - ValidateVideoProfile(profile); - assert(false); - } catch (const GateFailure& error) { - assert(error.stage() == "argument-profile"); + for (const SIZE size : {SIZE{1920, 1200}, SIZE{720, 1280}, SIZE{4, 4}}) { + ValidateVideoProfile(VideoProfile{static_cast(size.cx), static_cast(size.cy), 30, 3'000'000}); + } + for (const SIZE size : {SIZE{0, 720}, SIZE{2, 720}, SIZE{1280, 2}, SIZE{1279, 720}, + SIZE{1280, 719}, SIZE{2562, 1440}, SIZE{2560, 1442}, SIZE{3840, 2160}}) { + try { + ValidateVideoProfile(VideoProfile{static_cast(size.cx), static_cast(size.cy), 30, 3'000'000}); + assert(false); + } catch (const GateFailure& error) { + assert(error.stage() == "argument-profile"); + } } CheckEncoderCandidates(); CheckH264ProfileCompatibility(); From eca6ea88b9c3d615d974f9a2b0d44e356ef3990f Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:35:53 +0800 Subject: [PATCH 071/262] fix(capture): preserve source geometry through Linux scaling --- docs/research/native-client-media.md | 11 ++++- docs/standards/media-quality.md | 10 ++-- docs/status.md | 4 +- docs/todo.md | 6 +-- native/capture/linux/README.md | 10 +++- native/capture/linux/main.c | 40 +++++++++++++--- native/capture/linux/main.test.c | 70 +++++++++++++++++++++++++++- 7 files changed, 131 insertions(+), 20 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 441259ecb..9b24329e0 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -536,7 +536,16 @@ ceiling. A 640x480 source stays 640x480; portrait and ultrawide sources no longe require an encoded landscape canvas. The converter, adaptation owner, output generation and frame envelope are reused. Axis-specific limits still apply; this does not yet give portrait capture a transposed ceiling or add a 4K setting. -Browser and other platform geometry remain separate pending work. +Browser and macOS geometry remain separate pending work. + +Linux uses the existing [GStreamer scaler](https://gstreamer.freedesktop.org/documentation/videoconvertscale/videoscale.html) +with bounded even-dimension ranges instead of fixed output dimensions. Square +pixel negotiation retains presentation aspect without a padded canvas; encoded +samples provide the actual dimensions for the frame envelope. The headless +control changes portrait, ultrawide, small, odd-sized and square inputs within +one pipeline, checks the output ceiling and image corners, and exercises the +smallest derived output. Compilation and those regressions passed on Linux; +Portal capture and hardware encoder resizing still need a physical device. Local synthetic-input controls exercised that production worker through four successive source sizes: 1080x1920, 3440x1440, 640x480 and 1920x1080. Under the diff --git a/docs/standards/media-quality.md b/docs/standards/media-quality.md index 2ab476242..747d746a3 100644 --- a/docs/standards/media-quality.md +++ b/docs/standards/media-quality.md @@ -42,10 +42,12 @@ contract. [Status](../status.md) indexes delivery and remaining acceptance. frame matches its desktop source dimensions. Other frames retain their own aspect ratio. This changes only the encoded presentation, never the game or display settings; vendor-private scaling is not inferred. - Each Windows output fits that presentation within its selected ceiling using - even dimensions, without enlarging the source or encoding a fixed landscape - canvas. The existing WebRTC encoder may adapt further; source resizing does - not create a new room or route. Frame envelopes carry actual output dimensions. +- Windows and Linux outputs preserve source presentation aspect within the + selected ceiling, rounded to even dimensions. Small sources are not enlarged + to a fixed landscape canvas. Windows uses its existing converter; Linux uses + GStreamer's size negotiation. Source resizing does not create a new room or + route, and frame envelopes carry actual output dimensions. Network adaptation + remains owned by the existing media framework. - Windows SDR conversion owns one output color contract: limited-range BT.601 NV12 with matching H.264 metadata. RGB capture and decoded video retain their input range/matrix through conversion and relay scaling; dimensions do not diff --git a/docs/status.md b/docs/status.md index b0158532a..dd2090f09 100644 --- a/docs/status.md +++ b/docs/status.md @@ -65,8 +65,8 @@ Numeric audio is implemented in the unpublished candidate: Browser/server v24 and Native control v10 replace the preset field; capture v7 and SQLite schema 2 remain unchanged. Matched candidate App/Web/Server builds are required; 1.x participants are rejected by the existing compatibility checks. -Windows output now preserves the source shape within the existing ceiling; -axis-independent limits, other-platform geometry, 4K and additional codecs are +Windows and Linux output now preserve source shape within the existing ceiling; +axis-independent limits, Browser/macOS geometry, 4K and additional codecs are still assessed through the media boundary. Native Magicsock remains an isolated experiment with incomplete product integration and unresolved recovery cases. These proposals do not yet change diff --git a/docs/todo.md b/docs/todo.md index b0a79896c..4f25a088a 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -50,9 +50,9 @@ Implementation needs a concrete useful result within the product model. actual output. Numeric audio admission, storage and the compact custom input implement [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s use case, with local Browser/direct/relay and Browser/native SFU evidence. Next assess - source-aspect-preserving geometry and 4K through the same owners. Windows - output now fits the source without a padded landscape canvas or upscaling; - axis-independent limits, Browser/other-platform geometry and product 4K + source-aspect-preserving geometry and 4K through the same owners. Windows and + Linux outputs now fit the source without a fixed padded landscape canvas; + axis-independent limits, Browser/macOS geometry and product 4K admission remain open. Preserve safety limits and framework-owned adaptation. The [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment) diff --git a/native/capture/linux/README.md b/native/capture/linux/README.md index 6428e97c5..814348841 100644 --- a/native/capture/linux/README.md +++ b/native/capture/linux/README.md @@ -20,10 +20,11 @@ sh native/capture/linux/build.sh /outside/repository/build ``` Run `npm run check:native` on Linux to compile the sidecar, run the headless -output-profile, segment-timestamp and retirement regressions, and validate its probe/source list. +output-profile, source-geometry, segment-timestamp and retirement regressions, +and validate its probe/source list. The same check runs when the Linux CI job packages an App candidate. For the native build and regression alone, add `--check` to the command above. The test -uses GStreamer's core synthetic elements and needs no display, Portal session, +uses synthetic input and GStreamer's core/base elements and needs no display, Portal session, capture device or hardware encoder; physical capture remains a separate check. Input ticks, output activation and encoded frames use GStreamer running time, @@ -32,6 +33,11 @@ Keyframe events retain raw PTS for their timestamp field and normalized running time for their running-time field. The Portal stream uses the pipeline clock so unchanged content does not stop output keepalives. +GStreamer fits each source within the output's even-dimension ceilings while +preserving presentation aspect. It renegotiates those dimensions on source +resize; the encoded sample's caps supply the actual frame size. Requested +profile dimensions remain configuration facts, not measured output. + Each output bin contains its downstream flow failures before they reach the shared tee. The main-context bus handler reports that output as unavailable and closes its valve before stopping the branch. Failed outputs remain retired for diff --git a/native/capture/linux/main.c b/native/capture/linux/main.c index 5315881e5..a0baf05fa 100644 --- a/native/capture/linux/main.c +++ b/native/capture/linux/main.c @@ -941,6 +941,17 @@ static GstFlowReturn video_sample(GstAppSink *sink, gpointer data) { CaptureRun *run = output->run; GstSample *sample = gst_app_sink_pull_sample(sink); if (sample == NULL) return GST_FLOW_EOS; + GstCaps *caps = gst_sample_get_caps(sample); + const GstStructure *format = caps == NULL ? NULL : gst_caps_get_structure(caps, 0); + gint width = 0, height = 0; + if (format == NULL || !gst_structure_get_int(format, "width", &width) || + !gst_structure_get_int(format, "height", &height) || + !valid_video_size(width, height) || + (guint)width > output->profile.width || (guint)height > output->profile.height) { + gst_sample_unref(sample); + fail_output(output, "GStreamer returned H.264 dimensions outside the output ceiling"); + return GST_FLOW_ERROR; + } GstBuffer *buffer = gst_sample_get_buffer(sample); GstMapInfo mapped = GST_MAP_INFO_INIT; if (buffer == NULL || !gst_buffer_map(buffer, &mapped, GST_MAP_READ) || @@ -982,7 +993,7 @@ static GstFlowReturn video_sample(GstAppSink *sink, gpointer data) { return GST_FLOW_OK; } gboolean written = write_frame(2, recovery ? 1 : 0, (guint8)output->layer, - output->profile.width, output->profile.height, + width, height, timestamp / 100, 10000000ULL / output->profile.frame_rate, mapped.data, mapped.size); @@ -1099,6 +1110,22 @@ static int run_pipeline(CaptureRun *run, const char *starting_status) { return 0; } +static void append_video_scale(GString *pipeline, const VideoProfile *profile) { + // Let GStreamer preserve source aspect and renegotiate after a resize. The + // step keeps NV12 dimensions even; the profile is a ceiling, not a canvas. + g_string_append(pipeline, "videoconvert ! videoscale add-borders=false ! " + "video/x-raw,format=NV12,"); + const struct { const char *name; guint limit; } dimensions[] = { + {"width", profile->width}, {"height", profile->height}}; + for (guint index = 0; index < G_N_ELEMENTS(dimensions); ++index) { + // GStreamer represents a one-value range as a scalar. + g_string_append_printf(pipeline, dimensions[index].limit == 2 + ? "%s=(int)%u," : "%s=(int)[2,%u,2],", + dimensions[index].name, dimensions[index].limit); + } + g_string_append(pipeline, "pixel-aspect-ratio=1/1 ! "); +} + static int capture_video(int count, char **values) { CaptureRun run = {.last_input_timestamp = GST_CLOCK_TIME_NONE}; gboolean encoded = count > 1 && strcmp(values[1], "--encoded-video") == 0; @@ -1141,16 +1168,17 @@ static int capture_video(int count, char **values) { "frames. ! valve name=gate%u drop=%s ! ( " "queue name=raw%u max-size-buffers=1 max-size-bytes=0 " "max-size-time=0 leaky=downstream ! videorate drop-only=true ! " - "video/x-raw,framerate=%u/1 ! videoconvert ! videoscale ! " - "video/x-raw,format=NV12,width=%u,height=%u ! " + "video/x-raw,framerate=%u/1 ! ", + index, run.outputs[index].enabled ? "false" : "true", + index, profile->frame_rate); + append_video_scale(pipeline_text, profile); + g_string_append_printf(pipeline_text, "%s name=encoder%u ! video/x-h264,profile=constrained-baseline ! " "h264parse config-interval=-1 ! " "video/x-h264,stream-format=byte-stream,alignment=au ! " "appsink name=output%u emit-signals=true sync=false async=false " "max-buffers=1 drop=false ) ", - index, run.outputs[index].enabled ? "false" : "true", - index, profile->frame_rate, profile->width, profile->height, encoder_info->factory, - index, index); + encoder_info->factory, index, index); } GstElement *pipeline = gst_parse_launch(pipeline_text->str, &error); g_string_free(pipeline_text, TRUE); diff --git a/native/capture/linux/main.test.c b/native/capture/linux/main.test.c index 1052c9b82..33a60e967 100644 --- a/native/capture/linux/main.test.c +++ b/native/capture/linux/main.test.c @@ -1,7 +1,8 @@ -// build.sh --check uses synthetic GStreamer elements; no capture or codec is used. +// build.sh --check uses synthetic video/GStreamer elements; no capture or codec is used. #define main capture_main #include "main.c" #undef main +#include typedef struct { GstElement parent; guint bitrate; } FixtureEncoder; typedef struct { GstElementClass parent; } FixtureEncoderClass; @@ -209,7 +210,8 @@ static void check_encoder_timestamp_segment(GstClockTime offset) { gst_pad_add_probe(source_pad, GST_PAD_PROBE_TYPE_BUFFER, begin_input_frame, &run, NULL); gst_pad_add_probe(source_pad, GST_PAD_PROBE_TYPE_BUFFER, output_input_frame, &output, NULL); gst_pad_add_probe(sink_pad, GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, record_key_request, &request, NULL); - GstCaps *caps = gst_caps_from_string("video/x-h264,stream-format=byte-stream,alignment=au"); + GstCaps *caps = gst_caps_from_string( + "video/x-h264,stream-format=byte-stream,alignment=au,width=640,height=480"); GstBuffer *buffer = gst_buffer_new_allocate(NULL, sizeof(h264), NULL); gst_buffer_fill(buffer, 0, h264, sizeof(h264)); GST_BUFFER_PTS(buffer) = offset + GST_SECOND; @@ -244,6 +246,8 @@ static void check_encoder_timestamp_segment(GstClockTime offset) { g_assert_cmpint(bytes[37], ==, 2); g_assert_cmpint(bytes[38], ==, 1); g_assert_cmpuint(read_be((guint8 *)bytes + 40, 8), ==, 3 * GST_SECOND / 100); + g_assert_cmpuint(read_be((guint8 *)bytes + 56, 2), ==, 640); + g_assert_cmpuint(read_be((guint8 *)bytes + 58, 2), ==, 480); g_assert_cmpmem(bytes + 64, sizeof(h264), h264, sizeof(h264)); g_assert_cmpuint(output.activation_timestamp, ==, 3 * GST_SECOND); g_assert_nonnull(request); @@ -274,6 +278,66 @@ static GstState branch_state(VideoOutput *output) { return state; } +static void check_source_geometry(guint max_width, guint max_height) { + const VideoProfile profile = {.width = max_width, .height = max_height}; + GString *description = g_string_new("appsrc name=source format=time ! "); + append_video_scale(description, &profile); + g_string_append(description, "appsink name=output sync=false async=false"); + GError *error = NULL; + GstElement *pipeline = gst_parse_launch(description->str, &error); + g_string_free(description, TRUE); + g_assert_no_error(error); + GstElement *source = gst_bin_get_by_name(GST_BIN(pipeline), "source"); + GstElement *sink = gst_bin_get_by_name(GST_BIN(pipeline), "output"); + g_assert_cmpint(gst_element_set_state(pipeline, GST_STATE_PLAYING), !=, + GST_STATE_CHANGE_FAILURE); + // Keep one pipeline while input caps change, as a resized captured window or + // decoded relay source does. Small sources must not be enlarged to the ceiling. + const guint sizes[][2] = {{1080, 1920}, {3440, 1440}, {640, 480}, + {1920, 1080}, {641, 479}, {600, 600}}; + for (guint index = 0; index < G_N_ELEMENTS(sizes); ++index) { + GstVideoInfo input; + g_assert_true(gst_video_info_set_format(&input, GST_VIDEO_FORMAT_NV12, + sizes[index][0], sizes[index][1])); + GstCaps *caps = gst_video_info_to_caps(&input); + gst_app_src_set_caps(GST_APP_SRC(source), caps); + gst_caps_unref(caps); + GstBuffer *buffer = gst_buffer_new_allocate(NULL, input.size, NULL); + gst_buffer_memset(buffer, 0, 128, input.size); + gst_buffer_memset(buffer, 0, 235, input.offset[1]); + GST_BUFFER_PTS(buffer) = index * GST_SECOND / 30; + GST_BUFFER_DURATION(buffer) = GST_SECOND / 30; + g_assert_cmpint(gst_app_src_push_buffer(GST_APP_SRC(source), buffer), ==, GST_FLOW_OK); + GstSample *sample = gst_app_sink_try_pull_sample(GST_APP_SINK(sink), GST_SECOND); + g_assert_nonnull(sample); + GstVideoInfo actual; + g_assert_true(gst_video_info_from_caps(&actual, gst_sample_get_caps(sample))); + g_assert_true(valid_video_size(actual.width, actual.height)); + g_assert_cmpuint(actual.width, <=, max_width); + g_assert_cmpuint(actual.height, <=, max_height); + g_assert_cmpint(actual.par_n, ==, actual.par_d); + double scale = MIN(1.0, MIN((double)max_width / input.width, + (double)max_height / input.height)); + g_assert_cmpfloat(ABS(actual.width - input.width * scale), <=, 2); + g_assert_cmpfloat(ABS(actual.height - input.height * scale), <=, 2); + GstVideoFrame frame; + g_assert_true(gst_video_frame_map(&frame, &actual, gst_sample_get_buffer(sample), GST_MAP_READ)); + const guint8 *luma = GST_VIDEO_FRAME_PLANE_DATA(&frame, 0); + const gint stride = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 0); + // No encoded letterbox: even the four corners still contain the source. + for (gint y = 0; y < actual.height; y += actual.height - 1) { + g_assert_cmpint(luma[y * stride], >=, 230); + g_assert_cmpint(luma[y * stride + actual.width - 1], >=, 230); + } + gst_video_frame_unmap(&frame); + gst_sample_unref(sample); + } + gst_element_set_state(pipeline, GST_STATE_NULL); + gst_object_unref(source); + gst_object_unref(sink); + gst_object_unref(pipeline); +} + static void check_slot_retirement(void) { CaptureRun run = {.output_count = 6}; g_mutex_init(&run.lock); @@ -470,6 +534,8 @@ int main(int argc, char **argv) { check_encoder_admission_and_control(); check_bus_error_retains_primary_cause(); check_slot_profiles(); + check_source_geometry(1920, 1080); + check_source_geometry(2, 2); check_encoder_timestamp_segment(1000ULL * 60 * 60 * GST_SECOND); check_encoder_timestamp_segment(0); check_slot_retirement(); From 9dec156cdb22258c22f1643690a7d88ceafe23c2 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:04:55 +0800 Subject: [PATCH 072/262] feat(media)!: accept numeric video size ceilings Replace resolution preset strings with bounded dimensions in the unpublished 2.0 contract. Keep UI shortcuts and implementation limits separate, reuse existing media owners, and verify a live custom size through Windows native capture. --- docs/research/native-client-media.md | 20 ++- docs/standards/media-quality.md | 8 +- docs/status.md | 8 +- docs/todo.md | 4 +- internal/app/app.go | 4 + .../app/config/quality_preference_test.go | 10 +- internal/app/loopback/server.go | 22 +-- .../nativecontrol/operation_failure_test.go | 8 +- internal/app/nativecontrol/session.go | 15 +- internal/app/nativecontrol/session_test.go | 26 ++-- internal/server/protocol/constants.go | 2 +- internal/server/protocol/contract_test.go | 6 +- internal/server/protocol/types.go | 37 +++-- internal/server/signal/server_test.go | 12 +- scripts/app-media-gate.ts | 6 +- scripts/app-native-host-gate.ts | 21 ++- scripts/embedded-sfu-page.ts | 4 +- scripts/peer-assisted-benchmark.ts | 5 +- .../components/living/QualityPresets.tsx | 66 ++++++++- src/client/locales/en.ts | 3 + src/client/locales/zh.ts | 3 + src/client/media/browser-encoding-pool.ts | 4 +- src/client/media/quality.ts | 45 +++--- src/client/native/wire.ts | 3 +- src/client/pages/ControlsPreview.tsx | 11 +- src/client/pages/HostPage.tsx | 38 ++--- src/client/sfu/publisher.ts | 5 +- src/client/styles.css | 5 + src/client/webrtc/video-codec-preflight.ts | 3 +- src/shared/protocol.ts | 21 +-- tests/browser-encoding-pool.test.ts | 2 +- tests/client-state.test.ts | 6 +- tests/fixtures/wire-samples.json | 139 ++++++++++++++---- tests/host-peer-tracks.test.ts | 2 +- tests/host-quality-ownership.test.ts | 4 +- tests/native-client.test.ts | 2 +- tests/protocol.test.ts | 6 +- tests/quality-preference.test.ts | 2 +- tests/quality.test.ts | 23 ++- 39 files changed, 410 insertions(+), 201 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 9b24329e0..1e8c965b6 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -513,7 +513,7 @@ a corrected SDP interpretation, not evidence that all Firefox sharing lost audio Planning direction reviewed 2026-10-09 for 2.0; the design below is a proposal, not a replacement for the current contract or profile decision above. The owner reopened numeric media requests and capability negotiation as one coherent -extension. Resolution enums are enforced +extension. The published resolution enums are enforced by both signaling readers and Native control, while capture admission separately bounds dimensions. Native capture, forwarding and SFU declarations also admit specific encoded formats. Removing one validator would leave these consumers @@ -569,10 +569,20 @@ The candidate's Native capture admission now shares the numeric dimension bounds already used by encoded outputs instead of duplicating the four UI presets. Even dimensions must fit within 2560x1440; a source ceiling must also yield a valid half-size output. Frame-rate, bitrate and output-count bounds -remain unchanged. Signaling still uses the existing resolution enum: custom -dimensions are an internal foundation, not yet a product control or a 4K option. -Device capability and the public request contract still need separate admission; -removing an enum is not permission to allocate arbitrary media resources. +remain unchanged. Candidate signaling and Native control now carry numeric +width/height ceilings, with custom input beside the existing UI shortcuts. +Descriptive wire bounds are independent of the current implementation maximum; +App discovery advertises that maximum, and Native rejects unsupported profiles +as operation failures without retiring the control session. This does not yet +admit product 4K. Removing an enum is not permission to allocate arbitrary media +resources. + +The Windows H.264 UI gate applies a non-preset 960x600 ceiling during a live +share and observes fitted output without replacing the Viewer media object. +Preset changes, paused updates, source recovery/replacement and subsequent +sharing still pass. Value-based comparison prevents an identical deserialized +size from turning an audio-only edit into a video update. Numeric preference +round-trips and malformed/unsupported requests have separate boundary checks. The proposed boundary follows the [Media Capture specification](https://www.w3.org/TR/mediacapture-streams/#constrainable-interface): diff --git a/docs/standards/media-quality.md b/docs/standards/media-quality.md index 747d746a3..8e713f29d 100644 --- a/docs/standards/media-quality.md +++ b/docs/standards/media-quality.md @@ -126,8 +126,12 @@ The three recommended profiles are ceilings, not delivery guarantees: | `1080p30` | 1920x1080 at 30 fps | 5 Mbps | | `1080p60` | 1920x1080 at 60 fps | 8 Mbps | -`1080p30` is the default. Advanced resolution includes 480p, 720p, 1080p, and -1440p; frame rate, bitrate, and `maintain-resolution | balanced | +`1080p30` is the default. The 2.0 candidate carries numeric width/height ceilings; +480p, 720p, 1080p and 1440p remain UI shortcuts. Advanced picture settings also +accept custom even dimensions within the current 2560x1440 implementation bound. +Width and height form one applied edit. Larger wire-valid requests do not +authorize an adapter to exceed its local capture, codec or allocation limits. +Frame rate, bitrate, and `maintain-resolution | balanced | maintain-framerate` remain independent controls. Display video uses the standard `contentHint = "motion"` for game motion. Native H264 retains Constrained Baseline. Capture validates the coding-tool subset under diff --git a/docs/status.md b/docs/status.md index dd2090f09..6039e2cf8 100644 --- a/docs/status.md +++ b/docs/status.md @@ -61,11 +61,13 @@ one hour without authenticated participants; the private deployment retains the default. Operator records own activation, database and postflight evidence. The owner is holding the unpublished maintenance changes for 2.0. Preparation includes separating media presets from numeric requests and bounded capabilities. -Numeric audio is implemented in the unpublished candidate: Browser/server v24 -and Native control v10 replace the preset field; capture v7 and SQLite schema 2 +Numeric audio and video ceilings are implemented in the unpublished candidate: +Browser/server v24 and Native control v10 replace the preset fields; +capture v7 and SQLite schema 2 remain unchanged. Matched candidate App/Web/Server builds are required; 1.x participants are rejected by the existing compatibility checks. -Windows and Linux output now preserve source shape within the existing ceiling; +Custom width/height controls retain the current implementation ceiling. +Windows and Linux output now preserve source shape within that ceiling; axis-independent limits, Browser/macOS geometry, 4K and additional codecs are still assessed through the media boundary. Native Magicsock remains an isolated experiment with incomplete product diff --git a/docs/todo.md b/docs/todo.md index 4f25a088a..b5f27e894 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -49,7 +49,9 @@ Implementation needs a concrete useful result within the product model. separate UI presets, numeric requests, bounded implementation capabilities and actual output. Numeric audio admission, storage and the compact custom input implement [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s use case, with - local Browser/direct/relay and Browser/native SFU evidence. Next assess + local Browser/direct/relay and Browser/native SFU evidence. Numeric video + requests and custom width/height controls now pass the existing Windows live + capture gate within current implementation limits. Next assess source-aspect-preserving geometry and 4K through the same owners. Windows and Linux outputs now fit the source without a fixed padded landscape canvas; axis-independent limits, Browser/macOS geometry and product 4K diff --git a/internal/app/app.go b/internal/app/app.go index 63b748e07..2e38d3430 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -27,6 +27,7 @@ import ( "github.com/TNTcraftHIM/Piik/internal/diagnostics" serverapp "github.com/TNTcraftHIM/Piik/internal/server/app" serverconfig "github.com/TNTcraftHIM/Piik/internal/server/config" + "github.com/TNTcraftHIM/Piik/internal/server/protocol" "github.com/TNTcraftHIM/Piik/internal/server/webassets" "golang.org/x/mod/semver" ) @@ -627,6 +628,9 @@ func discoverNativeMedia(ctx context.Context, configuredPath string) nativeRunti captureProcess: path, capture: capabilities, capabilities: loopback.NativeMediaCapabilities{ + MaxVideoResolution: protocol.QualityResolution{ + Width: nativecapture.MaxVideoWidth, Height: nativecapture.MaxVideoHeight, + }, MaxAudioBitrate: nativeaudio.MaxSendBitrate, ReceiverReuse: true, Video: summary.Video, diff --git a/internal/app/config/quality_preference_test.go b/internal/app/config/quality_preference_test.go index a9c6dcb8f..71b00c366 100644 --- a/internal/app/config/quality_preference_test.go +++ b/internal/app/config/quality_preference_test.go @@ -20,7 +20,7 @@ func TestQualityPreferencePersistsWithoutChangingAppConfiguration(t *testing.T) t.Fatalf("initial preference: %v, %v", p.Read(), err) } quality := protocol.DefaultQualitySettings - quality.Resolution, quality.MaxAudioBitrate = "1440p", 320_000 + quality.Resolution, quality.MaxAudioBitrate = protocol.QualityResolution{Width: 1920, Height: 1200}, 320_000 if err := p.Save(quality); err != nil { t.Fatal(err) } @@ -29,7 +29,7 @@ func TestQualityPreferencePersistsWithoutChangingAppConfiguration(t *testing.T) t.Fatalf("restored: %v, %v", loaded.Read(), err) } copy := loaded.Read() - copy.Resolution = "480p" + copy.Resolution.Width = 854 if *loaded.Read() != quality { t.Fatal("caller modified stored settings") } @@ -38,7 +38,7 @@ func TestQualityPreferencePersistsWithoutChangingAppConfiguration(t *testing.T) t.Fatal("preference changed strict App configuration") } invalid := quality - invalid.Resolution = "2160p" + invalid.Resolution.Width = 8194 if p.Save(invalid) == nil || *p.Read() != quality { t.Fatal("invalid choice changed preference") } @@ -48,8 +48,8 @@ func TestQualityPreferencePersistsWithoutChangingAppConfiguration(t *testing.T) if err := os.Mkdir(path+".quality", 0700); err != nil { t.Fatal(err) } - quality.Resolution = "720p" - if p.Save(quality) == nil || p.Read().Resolution != "1440p" { + quality.Resolution = protocol.QualityResolution{Width: 1280, Height: 720} + if p.Save(quality) == nil || p.Read().Resolution != (protocol.QualityResolution{Width: 1920, Height: 1200}) { t.Fatal("failed write replaced saved preference") } } diff --git a/internal/app/loopback/server.go b/internal/app/loopback/server.go index 5a888f30e..e7617ac41 100644 --- a/internal/app/loopback/server.go +++ b/internal/app/loopback/server.go @@ -17,6 +17,7 @@ import ( "time" appconfig "github.com/TNTcraftHIM/Piik/internal/app/config" + "github.com/TNTcraftHIM/Piik/internal/server/protocol" "github.com/coder/websocket" ) @@ -72,16 +73,17 @@ type Health struct { } type NativeMediaCapabilities struct { - MaxAudioBitrate int `json:"maxAudioBitrate,omitempty"` - ReceiverReuse bool `json:"receiverReuse,omitempty"` - Video bool `json:"video"` - CaptureBorderControl bool `json:"captureBorderControl,omitempty"` - Microphone bool `json:"microphone,omitempty"` - ProcessAudio bool `json:"processAudio"` - ProcessAudioExclusion bool `json:"processAudioExclusion,omitempty"` - SystemAudio bool `json:"systemAudio"` - HardwareH264 bool `json:"hardwareH264"` - SoftwareVP8 bool `json:"softwareVP8"` + MaxVideoResolution protocol.QualityResolution `json:"maxVideoResolution,omitzero"` + MaxAudioBitrate int `json:"maxAudioBitrate,omitempty"` + ReceiverReuse bool `json:"receiverReuse,omitempty"` + Video bool `json:"video"` + CaptureBorderControl bool `json:"captureBorderControl,omitempty"` + Microphone bool `json:"microphone,omitempty"` + ProcessAudio bool `json:"processAudio"` + ProcessAudioExclusion bool `json:"processAudioExclusion,omitempty"` + SystemAudio bool `json:"systemAudio"` + HardwareH264 bool `json:"hardwareH264"` + SoftwareVP8 bool `json:"softwareVP8"` } type Server struct { diff --git a/internal/app/nativecontrol/operation_failure_test.go b/internal/app/nativecontrol/operation_failure_test.go index 4a69ba5bd..e4f2b3b19 100644 --- a/internal/app/nativecontrol/operation_failure_test.go +++ b/internal/app/nativecontrol/operation_failure_test.go @@ -138,7 +138,7 @@ func TestOperationalFailureAndIdempotentStopKeepTheRealControlConnection(t *test defer connection.CloseNow() for index, command := range []struct{ kind, fields, expected string }{ {"hello", "", "ready"}, - {"start-share", `,"shareId":"missing_share","source":{"kind":"display","sourceId":"1","title":"Fixture"},"edgeCapacity":1,"codec":"vp8","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}`, "request-failed"}, + {"start-share", `,"shareId":"missing_share","source":{"kind":"display","sourceId":"1","title":"Fixture"},"edgeCapacity":1,"codec":"vp8","profile":{"resolution":{"width":1920,"height":1080},"maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}`, "request-failed"}, {"prepare-edge", `,"shareId":"missing_share","connectionId":"edge_123456","iceServers":[]`, "request-failed"}, {"edge-answer", `,"shareId":"missing_share","connectionId":"edge_123456","sdp":"v=0"`, "request-failed"}, {"receive-candidate", `,"shareId":"missing_share","connectionId":"edge_123456","candidate":null`, "request-failed"}, @@ -180,7 +180,7 @@ func TestFailedEdgeAndLateStopDoNotRetireAnotherShare(t *testing.T) { t.Setenv("PIIK_QUIET_CAPTURE_FIXTURE", t.TempDir()) session := New(executable, nativecapture.Capabilities{SoftwareVP8: true}, false) defer session.Close() - const start = `{"version":10,"id":"request_start","type":"start-share","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"audio":false,"edgeCapacity":1,"codec":"vp8","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}}` + const start = `{"version":10,"id":"request_start","type":"start-share","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"audio":false,"edgeCapacity":1,"codec":"vp8","profile":{"resolution":{"width":1920,"height":1080},"maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}}` if value, err := awaitControlResponse(t, session, []byte(start)); err != nil { t.Fatal(err) } else if _, ok := value.(shareStartedResponse); !ok { @@ -274,8 +274,8 @@ func TestCaptureStartupAndReplacementRemainCancellable(t *testing.T) { } session := New(executable, nativecapture.Capabilities{SoftwareVP8: true}, false) defer session.Close() - start := `{"version":10,"id":"request_start","type":"start-share","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"edgeCapacity":1,"codec":"vp8","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}}` - payload := strings.Replace(start, "1080p", "720p", 1) + start := `{"version":10,"id":"request_start","type":"start-share","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"edgeCapacity":1,"codec":"vp8","profile":{"resolution":{"width":1920,"height":1080},"maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}}` + payload := strings.Replace(start, `"width":1920,"height":1080`, `"width":1280,"height":720`, 1) if kind == "replace-share-source" { if value, err := awaitControlResponse(t, session, []byte(start)); err != nil { t.Fatal(err) diff --git a/internal/app/nativecontrol/session.go b/internal/app/nativecontrol/session.go index f3e8e3ab6..74c488593 100644 --- a/internal/app/nativecontrol/session.go +++ b/internal/app/nativecontrol/session.go @@ -641,25 +641,14 @@ func (session *Session) startShare( } func nativeQualityProfile(settings qualitySettings) nativehost.QualityProfile { - width, height := uint32(0), uint32(0) - switch settings.Resolution { - case "480p": - width, height = 854, 480 - case "720p": - width, height = 1280, 720 - case "1080p": - width, height = 1920, 1080 - case "1440p": - width, height = 2560, 1440 - } audioBitrate := int(settings.MaxAudioBitrate) if audioBitrate == 0 { audioBitrate = int(protocol.DefaultQualitySettings.MaxAudioBitrate) } return nativehost.QualityProfile{ Video: nativecapture.VideoProfile{ - Width: width, - Height: height, + Width: uint32(settings.Resolution.Width), + Height: uint32(settings.Resolution.Height), Framerate: uint32(settings.MaxFramerate), Bitrate: uint32(settings.MaxBitrate), Preference: settings.DegradationPreference, diff --git a/internal/app/nativecontrol/session_test.go b/internal/app/nativecontrol/session_test.go index de572e1e8..09e240589 100644 --- a/internal/app/nativecontrol/session_test.go +++ b/internal/app/nativecontrol/session_test.go @@ -18,6 +18,7 @@ import ( "github.com/TNTcraftHIM/Piik/internal/app/loopback" "github.com/TNTcraftHIM/Piik/internal/app/nativecapture" "github.com/TNTcraftHIM/Piik/internal/app/nativehost" + "github.com/TNTcraftHIM/Piik/internal/server/protocol" "github.com/pion/webrtc/v4" ) @@ -253,7 +254,7 @@ func TestMicrophoneOptInPreservesLegacyAndOrdersStateBeforeCompletion(t *testing request := startShareRequest{Version: 10, ID: "request_start", Type: "start-share", ShareID: "share_123456", Source: nativecapture.CaptureTarget{Kind: "display", SourceID: "1", Title: "Fixture"}, MicrophoneMixing: optIn, EdgeCapacity: 1, Codec: "vp8", - Profile: qualitySettings{Resolution: "1080p", MaxFramerate: 30, MaxBitrate: 5_000_000, DegradationPreference: "balanced"}} + Profile: qualitySettings{Resolution: protocol.QualityResolution{Width: 1920, Height: 1080}, MaxFramerate: 30, MaxBitrate: 5_000_000, DegradationPreference: "balanced"}} payload, _ := json.Marshal(request) result, err := awaitControlResponse(t, session, payload) started, ok := result.(shareStartedResponse) @@ -335,11 +336,11 @@ func TestCaptureBorderPreferenceFollowsSourceAndProfile(t *testing.T) { start := startShareRequest{Version: 10, ID: "request_start", Type: "start-share", ShareID: "share_123456", Source: nativecapture.CaptureTarget{Kind: "display", SourceID: "1", Title: "Fixture"}, ShowCaptureBorder: true, EdgeCapacity: 1, Codec: "vp8", - Profile: qualitySettings{Resolution: "1080p", MaxFramerate: 30, MaxBitrate: 5_000_000, DegradationPreference: "balanced"}} + Profile: qualitySettings{Resolution: protocol.QualityResolution{Width: 1920, Height: 1080}, MaxFramerate: 30, MaxBitrate: 5_000_000, DegradationPreference: "balanced"}} handle(start) assertBorder(supported) profile := start.Profile - profile.Resolution = "480p" + profile.Resolution = protocol.QualityResolution{Width: 854, Height: 480} handle(updateShareRequest{Version: 10, ID: "request_update", Type: "update-share", ShareID: start.ShareID, Profile: profile}) session.mu.Lock() done := session.updateDone @@ -393,9 +394,9 @@ func TestQuietHostUpdatePreservesControlAndCancelsCleanly(t *testing.T) { } return value } - const start = `{"version":10,"id":"request_start","type":"start-share","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"audio":false,"adapterIndex":0,"encoderIndex":0,"edgeCapacity":1,"codec":"vp8","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}}` + const start = `{"version":10,"id":"request_start","type":"start-share","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"audio":false,"adapterIndex":0,"encoderIndex":0,"edgeCapacity":1,"codec":"vp8","profile":{"resolution":{"width":1920,"height":1080},"maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}}` handle(start) - const update = `{"version":10,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"720p","maxFramerate":30,"maxBitrate":3000000,"degradationPreference":"balanced"}}` + const update = `{"version":10,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":{"width":1280,"height":720},"maxFramerate":30,"maxBitrate":3000000,"degradationPreference":"balanced"}}` if value := handle(update); value != nil { t.Fatalf("Host update did not defer: %#v", value) } @@ -652,7 +653,7 @@ func TestSourceEnumerationFailureDoesNotCloseControl(t *testing.T) { func TestQualitySettingsMapOnceIntoNativeMedia(t *testing.T) { settings := qualitySettings{ - Resolution: "1440p", MaxFramerate: 60, MaxBitrate: 12_000_000, + Resolution: protocol.QualityResolution{Width: 2560, Height: 1440}, MaxFramerate: 60, MaxBitrate: 12_000_000, DegradationPreference: "maintain-framerate", MaxAudioBitrate: 192000, } @@ -664,7 +665,11 @@ func TestQualitySettingsMapOnceIntoNativeMedia(t *testing.T) { profile.AudioBitrate != 192_000 { t.Fatalf("native profile = %+v", profile) } - settings.Resolution = "2160p" + settings.Resolution = protocol.QualityResolution{Width: 1920, Height: 1200} + if profile := nativeQualityProfile(settings); !profile.Valid() || profile.Video.Height != 1200 { + t.Fatalf("numeric size outside UI presets was not preserved: %+v", profile) + } + settings.Resolution = protocol.QualityResolution{Width: 3840, Height: 2160} if nativeQualityProfile(settings).Valid() { t.Fatal("unsupported resolution was accepted") } @@ -673,15 +678,18 @@ func TestQualitySettingsMapOnceIntoNativeMedia(t *testing.T) { func TestUpdateShareRequiresTheCurrentStrictProfile(t *testing.T) { session := New("missing-capture-process", nativecapture.Capabilities{}, false) t.Cleanup(func() { _ = session.Close() }) - valid := `{"version":10,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced","maxAudioBitrate":128000}}` + valid := `{"version":10,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":{"width":1920,"height":1080},"maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced","maxAudioBitrate":128000}}` assertOperationFailure(t, session, valid) // Opus permits this numeric request, but the current 20 ms sender does not. // Keep the control session usable and let the caller keep its current share. assertOperationFailure(t, session, strings.Replace(valid, `"maxAudioBitrate":128000`, `"maxAudioBitrate":510000`, 1)) + assertOperationFailure(t, session, strings.Replace(valid, + `"resolution":{"width":1920,"height":1080}`, + `"resolution":{"width":3840,"height":2160}`, 1)) if _, err := session.Handle(t.Context(), []byte(`{"version":10,"id":"request_options","type":"capture-options"}`)); err != nil { t.Fatalf("unsupported media setting retired control: %v", err) } - invalid := `{"version":10,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced","maxAudioBitrate":128000,"extra":true}}` + invalid := `{"version":10,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":{"width":1920,"height":1080},"maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced","maxAudioBitrate":128000,"extra":true}}` if _, err := session.Handle(t.Context(), []byte(invalid)); err == nil || err.Error() != "native update-share request is invalid" { t.Fatalf("extended update was accepted: %v", err) diff --git a/internal/server/protocol/constants.go b/internal/server/protocol/constants.go index cb393307b..2414043f6 100644 --- a/internal/server/protocol/constants.go +++ b/internal/server/protocol/constants.go @@ -64,7 +64,7 @@ var NatTraversalPaths = [...]string{"unknown", "ordinary", "predicted"} // DefaultQualitySettings mirrors DEFAULT_QUALITY_SETTINGS. var DefaultQualitySettings = QualitySettings{ - Resolution: "1080p", + Resolution: QualityResolution{Width: 1920, Height: 1080}, MaxFramerate: 30, MaxBitrate: 5_000_000, DegradationPreference: "balanced", diff --git a/internal/server/protocol/contract_test.go b/internal/server/protocol/contract_test.go index 206999b58..3ebd15393 100644 --- a/internal/server/protocol/contract_test.go +++ b/internal/server/protocol/contract_test.go @@ -117,7 +117,7 @@ func drop(base []member, keys ...string) []member { const ( testAssignmentNone = `{"upstream":{"kind":"none"},"childPeerIds":[],` + `"sfuPublicationGeneration":null}` - testQualitySettings = `{"resolution":"1080p","maxFramerate":60,"maxBitrate":8000000,` + + testQualitySettings = `{"resolution":{"width":1920,"height":1080},"maxFramerate":60,"maxBitrate":8000000,` + `"degradationPreference":"maintain-resolution"}` testRoutePolicy = `{"peerOnly":false,"topologyOptimization":true,"natPrediction":false}` testIceConfig = `{"iceServers":[],"natPredictionStunUrls":[]}` @@ -245,7 +245,7 @@ func TestAuthenticatedRoomRevision(t *testing.T) { object(drop(authenticatedHost(), "mediaMode")), false) assertServerMessage(t, "quality settings past the bitrate ceiling", object(with(authenticatedHost(), member{"qualitySettings", - `{"resolution":"1080p","maxFramerate":60,"maxBitrate":20000000,` + + `{"resolution":{"width":1920,"height":1080},"maxFramerate":60,"maxBitrate":20000000,` + `"degradationPreference":"maintain-resolution"}`})), false) } @@ -578,7 +578,7 @@ func TestDuplicateKeysKeepTheLastOccurrence(t *testing.T) { } // A duplicated object is replaced, not merged. if _, err := DecodeClientMessage( - []byte(`{"type":"set-quality-settings","qualitySettings":{"resolution":"480p"},` + + []byte(`{"type":"set-quality-settings","qualitySettings":{"resolution":{"width":854,"height":480}},` + `"qualitySettings":` + testQualitySettings + `}`)); err != nil { t.Fatalf("a duplicated object must be replaced wholesale: %v", err) } diff --git a/internal/server/protocol/types.go b/internal/server/protocol/types.go index e026f1424..f201cc158 100644 --- a/internal/server/protocol/types.go +++ b/internal/server/protocol/types.go @@ -35,7 +35,6 @@ var ( // Enumerations mirrored from src/shared/protocol.ts. var ( - qualityResolutions = []string{"480p", "720p", "1080p", "1440p"} degradationPreferences = []string{"maintain-resolution", "balanced", "maintain-framerate"} mediaRoutePhases = []string{"prepare", "active"} routeDemandReasons = []string{"join", "edge-unavailable", "parent-departed", "capacity-reduction", "sfu-bootstrap", "direct-convergence", "quality-convergence", "root-convergence"} @@ -105,13 +104,36 @@ func (v *MediaRouteUpstream) UnmarshalJSON(data []byte) error { // qualitySettingsSchema / routePolicySchema / runtimeCapabilitiesSchema // --------------------------------------------------------------------------- +// QualityResolution is a requested ceiling, not encoder availability or an allocation. +type QualityResolution struct { + Width Int `json:"width"` + Height Int `json:"height"` +} + +func (v *QualityResolution) UnmarshalJSON(data []byte) error { + *v = QualityResolution{} + type raw QualityResolution + present, err := decodeObject(data, (*raw)(v)) + if err != nil { + return err + } + if err := present.require("width", "height"); err != nil { + return err + } + if !inRangeInt(v.Width, 4, 8192) || v.Width%2 != 0 || + !inRangeInt(v.Height, 4, 8192) || v.Height%2 != 0 { + return errors.New("resolution is out of range") + } + return nil +} + // QualitySettings mirrors qualitySettingsSchema. type QualitySettings struct { - Resolution string `json:"resolution"` - MaxFramerate Int `json:"maxFramerate"` - MaxBitrate Int `json:"maxBitrate"` - DegradationPreference string `json:"degradationPreference"` - MaxAudioBitrate Int `json:"maxAudioBitrate,omitempty"` + Resolution QualityResolution `json:"resolution"` + MaxFramerate Int `json:"maxFramerate"` + MaxBitrate Int `json:"maxBitrate"` + DegradationPreference string `json:"degradationPreference"` + MaxAudioBitrate Int `json:"maxAudioBitrate,omitempty"` } // UnmarshalJSON implements json.Unmarshaler. @@ -129,9 +151,6 @@ func (v *QualitySettings) UnmarshalJSON(data []byte) error { if err := present.optional("maxAudioBitrate"); err != nil { return err } - if !enumOf(v.Resolution, qualityResolutions...) { - return fmt.Errorf("unknown resolution %q", v.Resolution) - } if !inRangeInt(v.MaxFramerate, 15, 60) { return errors.New("maxFramerate is out of range") } diff --git a/internal/server/signal/server_test.go b/internal/server/signal/server_test.go index 16ec7c23e..972767b2d 100644 --- a/internal/server/signal/server_test.go +++ b/internal/server/signal/server_test.go @@ -2430,13 +2430,13 @@ func TestSignalRetainsAuthoritativePauseWhenReconnectingHostAdvertisesUnpaused(t func TestSignalBindsNewShareQualityAndPreservesActiveShareQualityAcrossReconnect(t *testing.T) { h := startHarness(t, harnessOptions{}) initialSettings := map[string]any{ - "resolution": "1440p", + "resolution": protocol.QualityResolution{Width: 2560, Height: 1440}, "maxFramerate": 60, "maxBitrate": 8_000_000, "degradationPreference": "maintain-resolution", "maxAudioBitrate": 192000, } - initialJSON := `{"resolution":"1440p","maxFramerate":60,"maxBitrate":8000000,"degradationPreference":"maintain-resolution","maxAudioBitrate":192000}` + initialJSON := `{"resolution":{"width":2560,"height":1440},"maxFramerate":60,"maxBitrate":8000000,"degradationPreference":"maintain-resolution","maxAudioBitrate":192000}` firstGeneration := "first_share_generation_12345678" viewer := openClient(t, h) authenticate(t, viewer, h.room, protocol.RoleViewer, "quality-viewer-client", 1, "", presenceOptions{}) @@ -2448,13 +2448,13 @@ func TestSignalBindsNewShareQualityAndPreservesActiveShareQualityAcrossReconnect expectEqual(t, viewer.next("quality-settings").raw, `{"type":"quality-settings","qualitySettings":`+initialJSON+`}`) reconnectSettings := map[string]any{ - "resolution": "480p", + "resolution": protocol.QualityResolution{Width: 854, Height: 480}, "maxFramerate": 15, "maxBitrate": 2_000_000, "degradationPreference": "maintain-framerate", "maxAudioBitrate": 64000, } - reconnectJSON := `{"resolution":"480p","maxFramerate":15,"maxBitrate":2000000,"degradationPreference":"maintain-framerate","maxAudioBitrate":64000}` + reconnectJSON := `{"resolution":{"width":854,"height":480},"maxFramerate":15,"maxBitrate":2000000,"degradationPreference":"maintain-framerate","maxAudioBitrate":64000}` reconnectedHost := openClient(t, h) reconnectedAuth := authenticate(t, reconnectedHost, h.room, protocol.RoleHost, "quality-host-client", 1, firstGeneration, presenceOptions{qualitySettings: reconnectSettings}) @@ -2465,13 +2465,13 @@ func TestSignalBindsNewShareQualityAndPreservesActiveShareQualityAcrossReconnect } updatedSettings := map[string]any{ - "resolution": "720p", + "resolution": protocol.QualityResolution{Width: 1280, Height: 720}, "maxFramerate": 30, "maxBitrate": 3_000_000, "degradationPreference": "balanced", "maxAudioBitrate": 128000, } - updatedJSON := `{"resolution":"720p","maxFramerate":30,"maxBitrate":3000000,"degradationPreference":"balanced","maxAudioBitrate":128000}` + updatedJSON := `{"resolution":{"width":1280,"height":720},"maxFramerate":30,"maxBitrate":3000000,"degradationPreference":"balanced","maxAudioBitrate":128000}` reconnectedHost.sendJSON(map[string]any{"type": "set-quality-settings", "qualitySettings": updatedSettings}) expectEqual(t, viewer.next("quality-settings").raw, `{"type":"quality-settings","qualitySettings":`+updatedJSON+`}`) diff --git a/scripts/app-media-gate.ts b/scripts/app-media-gate.ts index 9bd1ca647..a4b296325 100644 --- a/scripts/app-media-gate.ts +++ b/scripts/app-media-gate.ts @@ -437,7 +437,7 @@ async function browserMediaGate(input: { encoderIndex: encoder.index, edgeCapacity: 2, profile: { - resolution: "1440p", + resolution: { width: 2560, height: 1440 }, maxFramerate: 30, maxBitrate: 3_000_000, degradationPreference: "maintain-resolution", @@ -597,7 +597,7 @@ async function browserMediaGate(input: { const liveUpdate = await request("update-share", { shareId, profile: { - resolution: "720p", + resolution: { width: 1280, height: 720 }, maxFramerate: 60, maxBitrate: 12_000_000, degradationPreference: "maintain-framerate", @@ -623,7 +623,7 @@ async function browserMediaGate(input: { const pausedUpdate = await request("update-share", { shareId, profile: { - resolution: "480p", + resolution: { width: 854, height: 480 }, maxFramerate: 15, maxBitrate: 2_000_000, degradationPreference: "maintain-resolution", diff --git a/scripts/app-native-host-gate.ts b/scripts/app-native-host-gate.ts index 494a7ab15..5c80f04a5 100644 --- a/scripts/app-native-host-gate.ts +++ b/scripts/app-native-host-gate.ts @@ -82,6 +82,7 @@ interface GateResult { qualityControlsEnabled: boolean; livePresetChanges: number; liveQualityChanged: boolean; + customQualityChanged: boolean; pausedQualityChanged: boolean; backgroundProfileRecovery: boolean | null; sourceWindowRecovery: boolean | null; @@ -632,6 +633,7 @@ async function main(): Promise { qualityControlsEnabled: false, livePresetChanges: 0, liveQualityChanged: false, + customQualityChanged: false, pausedQualityChanged: false, backgroundProfileRecovery: mode === "local" && sourceKind === "window" ? false : null, sourceWindowRecovery: mode === "local" && sourceKind === "window" ? false : null, @@ -1018,6 +1020,23 @@ async function main(): Promise { result.viewerWidth = viewerState.width; result.viewerHeight = viewerState.height; await assertVideoCodec(cdp, viewer, actualCodec); + stage = "native-custom-quality"; + const framesBeforeCustom = await evaluate(cdp, viewer, + `document.querySelector('video')?.getVideoPlaybackQuality().totalVideoFrames ?? 0`, Date.now() + 5_000); + await evaluate(cdp, host, `(() => { + const width = document.querySelector('.lr-video-size input[name="width"]'); + const height = document.querySelector('.lr-video-size input[name="height"]'); + width.focus(); width.value = '960'; + height.focus(); height.value = '600'; height.blur(); + })()`, Date.now() + 5_000); + result.customQualityChanged = await waitForValue( + deadline => evaluate(cdp!, viewer, `(() => { + const video = document.querySelector('video'); + return Boolean(video && window.__piikGateFitsProfile(960, 600) && + video.srcObject === window.__piikGateMedia && + video.getVideoPlaybackQuality().totalVideoFrames >= ${framesBeforeCustom + 10}); + })()`, deadline), Boolean, 20_000); + await assertVideoCodec(cdp, viewer, actualCodec); const framesBeforePreference = await evaluate( cdp, viewer, @@ -1474,7 +1493,7 @@ async function main(): Promise { result.viewerFrames >= 30 && result.viewerWidth > 0 && result.viewerWidth <= 2560 && result.viewerHeight > 0 && result.viewerHeight <= 1440 && result.codecPreserved : result.viewerConnected && result.viewerFrames >= 30 && - result.qualityControlsEnabled && result.liveQualityChanged && + result.qualityControlsEnabled && result.liveQualityChanged && result.customQualityChanged && result.backgroundProfileRecovery !== false && result.sourceWindowRecovery !== false && result.livePresetChanges === 2 && diff --git a/scripts/embedded-sfu-page.ts b/scripts/embedded-sfu-page.ts index 1efa47f29..13659cded 100644 --- a/scripts/embedded-sfu-page.ts +++ b/scripts/embedded-sfu-page.ts @@ -18,10 +18,10 @@ import { HostSfuRoute, type HostPublisherTransport } from "../src/client/media/h import { ViewerSfuRoute } from "../src/client/media/viewer-sfu-route"; const high: QualitySettings = { - resolution: "720p", maxFramerate: 15, maxBitrate: 2_000_000, + resolution: { width: 1280, height: 720 }, maxFramerate: 15, maxBitrate: 2_000_000, degradationPreference: "maintain-resolution", maxAudioBitrate: 64000, }; -const low: QualitySettings = { ...high, resolution: "480p" }; +const low: QualitySettings = { ...high, resolution: { width: 854, height: 480 } }; const shareGeneration = crypto.randomUUID(); const clientId = crypto.randomUUID(); let socket: WebSocket | null = null; diff --git a/scripts/peer-assisted-benchmark.ts b/scripts/peer-assisted-benchmark.ts index 2d951d313..444dda8b2 100644 --- a/scripts/peer-assisted-benchmark.ts +++ b/scripts/peer-assisted-benchmark.ts @@ -18,7 +18,6 @@ import { } from "../src/shared/media-copy-accounting"; import { QUALITY_PROFILES, - QUALITY_RESOLUTIONS, qualitySettingsEqual, type QualityProfileId, type QualitySettings, @@ -3203,7 +3202,7 @@ async function runCase( let routeTimingStatus: BenchmarkRun["routeTimingStatus"] = "not-requested"; try { const capture = PROFILE_SETTINGS[config.profileId]; - const captureResolution = QUALITY_RESOLUTIONS[capture.resolution]; + const captureResolution = capture.resolution; const commonInit = { width: captureResolution.width, height: captureResolution.height, @@ -3491,7 +3490,7 @@ async function main(): Promise { } const profile = PROFILE_SETTINGS[config.profileId]; - const profileResolution = QUALITY_RESOLUTIONS[profile.resolution]; + const profileResolution = profile.resolution; const report: BenchmarkReport = { schemaVersion: 5, startedAt: new Date().toISOString(), diff --git a/src/client/components/living/QualityPresets.tsx b/src/client/components/living/QualityPresets.tsx index 187a1d0b8..fe56be928 100644 --- a/src/client/components/living/QualityPresets.tsx +++ b/src/client/components/living/QualityPresets.tsx @@ -1,5 +1,10 @@ -import { QUALITY_PROFILES, QUALITY_PROFILE_KEYS, type QualityProfileId } from "../../media/quality"; +import { Fragment } from "react"; +import { + QUALITY_PROFILES, QUALITY_PROFILE_KEYS, QUALITY_RESOLUTIONS, qualityResolutionEqual, + type QualityProfileId, type QualityResolution, +} from "../../media/quality"; import { useCopy } from "../../ui/copy"; +import { Chip } from "./primitives"; import { QualityTileGlyph } from "./QualityTileGlyph"; import { Tooltip } from "./Tooltip"; @@ -18,10 +23,65 @@ export function QualityPresets({ selected, disabled = false, busy = false, onSel text={vis ? undefined : t("host.quality.title", { label: caption, mbps: (profile.maxBitrate / 1_000_000).toFixed(0) })}> ; })} ; } + +export function ResolutionControl({ value, maximum, disabled, onChange }: { + value: QualityResolution; + maximum: QualityResolution; + disabled: boolean; + onChange: (resolution: QualityResolution) => void; +}) { + const { t, vis } = useCopy(); + const axes = ["width", "height"] as const; + const reset = (container: HTMLElement) => { + for (const axis of axes) container.querySelector(`input[name="${axis}"]`)!.value = String(value[axis]); + }; + const apply = (container: HTMLElement) => { + const width = container.querySelector('input[name="width"]')!; + const height = container.querySelector('input[name="height"]')!; + if (![width, height].every(input => input.validity.valid && Number.isFinite(input.valueAsNumber))) { + reset(container); + return; + } + const next = { width: width.valueAsNumber, height: height.valueAsNumber }; + if (!qualityResolutionEqual(value, next)) onChange(next); + }; + return
{ + if (!event.currentTarget.contains(event.relatedTarget)) apply(event.currentTarget); + }} + onKeyDown={event => { + if (!(event.target instanceof HTMLInputElement) || (event.key !== "Enter" && event.key !== "Escape")) return; + event.preventDefault(); + event.stopPropagation(); + if (event.key === "Escape") reset(event.currentTarget); + event.target.blur(); + }}> +
+ {Object.entries(QUALITY_RESOLUTIONS).map(([label, resolution]) => maximum.width || resolution.height > maximum.height} + title={label} hint="hint-quality" onClick={() => onChange(resolution)}>{label})} +
+ + + {axes.map((axis, index) => + {index === 1 ? : null} + + )} + + + +
; +} diff --git a/src/client/locales/en.ts b/src/client/locales/en.ts index 301449aba..79a16c143 100644 --- a/src/client/locales/en.ts +++ b/src/client/locales/en.ts @@ -359,6 +359,9 @@ export const en: Record = { "host.advanced.preference.framerateHint": "Prefer frame rate", "host.advanced.audio": "Audio quality", "host.advanced.audioBitrate": "Audio bitrate limit", + "host.video.width": "Width limit in pixels", + "host.video.height": "Height limit in pixels", + "host.video.resolutionHint": "Width × height limits: {width} × {height} pixels. Source proportions are preserved.", "host.audio.bitrateHint": "Audio bitrate limit: {min}–{max} kbps.", "host.advanced.audio.saver": "Standard", "host.advanced.audio.music": "Music", diff --git a/src/client/locales/zh.ts b/src/client/locales/zh.ts index d3cbcc53c..62480a4fa 100644 --- a/src/client/locales/zh.ts +++ b/src/client/locales/zh.ts @@ -359,6 +359,9 @@ export const zh = { "host.advanced.preference.framerateHint": "优先帧率", "host.advanced.audio": "音频质量", "host.advanced.audioBitrate": "音频码率上限", + "host.video.width": "宽度上限(像素)", + "host.video.height": "高度上限(像素)", + "host.video.resolutionHint": "宽 × 高上限:{width} × {height} 像素。保留来源比例。", "host.audio.bitrateHint": "音频码率上限:{min}–{max} kbps。", "host.advanced.audio.saver": "普通", "host.advanced.audio.music": "音乐", diff --git a/src/client/media/browser-encoding-pool.ts b/src/client/media/browser-encoding-pool.ts index 97a92c161..de152ad5e 100644 --- a/src/client/media/browser-encoding-pool.ts +++ b/src/client/media/browser-encoding-pool.ts @@ -6,7 +6,7 @@ import { BrowserEncodingProducer } from "./browser-encoding-producer"; import { BROWSER_CARRIER_SIZE, type BrowserEncodingOutput } from "./browser-encoding-output"; import { codecIdentity, fits, nativeVideoBudget, negotiatedCodec, noRegression, observeVideo, projectEncodingMetrics, videoOutbound, type Observation } from "./browser-encoding-stats"; -import { QUALITY_RESOLUTIONS, videoQualitySettingsEqual, type QualityProfile } from "./quality"; +import { videoQualitySettingsEqual, type QualityProfile } from "./quality"; export interface BrowserPooledSender { updateProfile(profile: QualityProfile): Promise; @@ -63,7 +63,7 @@ export class BrowserEncodingPool { return { carrierScale: () => { if (!member.carrier) return; - const width = member.sender.track?.getSettings().width ?? QUALITY_RESOLUTIONS[member.profile.resolution].width; + const width = member.sender.track?.getSettings().width ?? member.profile.resolution.width; return Math.max(1, width / BROWSER_CARRIER_SIZE); }, updateProfile: async (next) => { diff --git a/src/client/media/quality.ts b/src/client/media/quality.ts index 584a12f4e..4a93b4c58 100644 --- a/src/client/media/quality.ts +++ b/src/client/media/quality.ts @@ -3,7 +3,6 @@ import { MIN_AUDIO_BITRATE, MAX_AUDIO_BITRATE, type DegradationPreference, - type QualityProfileId, type QualityResolution, type QualitySettings, } from "../../shared/protocol"; @@ -16,17 +15,18 @@ import { applySenderCaptureConstraints, senderCaptureTrack } from "./sender-vide export type { DegradationPreference, - QualityProfileId, QualityResolution, QualitySettings, } from "../../shared/protocol"; export type QualityProfile = QualitySettings; export const STARTUP_VIDEO_ENCODED_FRAMES = 5; +// Current product output boundary, independent of wire admission and UI shortcuts. +export const MAX_CAPTURE_RESOLUTION = { width: 2560, height: 1440 } as const; export const QUALITY_PROFILES = { "720p30": { - resolution: "720p", + resolution: { width: 1280, height: 720 }, maxFramerate: 30, maxBitrate: 3_000_000, degradationPreference: "balanced", @@ -34,13 +34,14 @@ export const QUALITY_PROFILES = { }, "1080p30": DEFAULT_QUALITY_SETTINGS, "1080p60": { - resolution: "1080p", + resolution: { width: 1920, height: 1080 }, maxFramerate: 60, maxBitrate: 8_000_000, degradationPreference: "balanced", maxAudioBitrate: 128000, }, -} as const satisfies Record; +} as const satisfies Record; +export type QualityProfileId = keyof typeof QUALITY_PROFILES; export const QUALITY_PROFILE_KEYS = { "720p30": "host.quality.720p30", @@ -49,14 +50,20 @@ export const QUALITY_PROFILE_KEYS = { } as const satisfies Record; export const QUALITY_RESOLUTIONS = { - "480p": { width: 854, height: 480, label: "480p" }, - "720p": { width: 1280, height: 720, label: "720p" }, - "1080p": { width: 1920, height: 1080, label: "1080p" }, - "1440p": { width: 2560, height: 1440, label: "1440p" }, -} as const satisfies Record< - QualityResolution, - { width: number; height: number; label: string } ->; + "480p": { width: 854, height: 480 }, + "720p": { width: 1280, height: 720 }, + "1080p": { width: 1920, height: 1080 }, + "1440p": { width: 2560, height: 1440 }, +} as const satisfies Record; + +export function qualityResolutionEqual(left: QualityResolution, right: QualityResolution): boolean { + return left.width === right.width && left.height === right.height; +} + +export function qualityResolutionLabel(resolution: QualityResolution): string { + return Object.entries(QUALITY_RESOLUTIONS).find(([, size]) => qualityResolutionEqual(size, resolution))?.[0] + ?? `${resolution.width}×${resolution.height}`; +} export const DEGRADATION_PREFERENCE_KEYS = { "maintain-resolution": "host.advanced.preference.resolution", @@ -127,7 +134,7 @@ export function videoQualitySettingsEqual( right: QualitySettings, ): boolean { return ( - left.resolution === right.resolution && + qualityResolutionEqual(left.resolution, right.resolution) && left.maxFramerate === right.maxFramerate && left.maxBitrate === right.maxBitrate && left.degradationPreference === right.degradationPreference @@ -150,7 +157,7 @@ export function matchingQualityProfileId( for (const id of Object.keys(QUALITY_PROFILES) as QualityProfileId[]) { const profile = QUALITY_PROFILES[id]; if ( - settings.resolution === profile.resolution && + qualityResolutionEqual(settings.resolution, profile.resolution) && settings.maxFramerate === profile.maxFramerate && settings.maxBitrate === profile.maxBitrate && settings.degradationPreference === profile.degradationPreference @@ -166,11 +173,11 @@ export function qualitySettingsLabel(settings: QualitySettings): string { if (profileId) { return say(QUALITY_PROFILE_KEYS[profileId]); } - return `${QUALITY_RESOLUTIONS[settings.resolution].label} ${settings.maxFramerate} fps · ${(settings.maxBitrate / 1_000_000).toFixed(1)} Mbps · ${say(DEGRADATION_PREFERENCE_KEYS[settings.degradationPreference])}`; + return `${qualityResolutionLabel(settings.resolution)} ${settings.maxFramerate} fps · ${(settings.maxBitrate / 1_000_000).toFixed(1)} Mbps · ${say(DEGRADATION_PREFERENCE_KEYS[settings.degradationPreference])}`; } function captureConstraints(profile: QualityProfile): MediaTrackConstraints { - const resolution = QUALITY_RESOLUTIONS[profile.resolution]; + const resolution = profile.resolution; return { width: { ideal: resolution.width, max: resolution.width }, height: { ideal: resolution.height, max: resolution.height }, @@ -219,7 +226,7 @@ export type BrowserCaptureSource = "browser" | "camera"; export async function captureBrowserSource(profile: QualityProfile, source: BrowserCaptureSource, deviceId = ""): Promise { if (source !== "camera") return captureDisplay(profile); - const resolution = QUALITY_RESOLUTIONS[profile.resolution]; + const resolution = profile.resolution; const stream = await navigator.mediaDevices.getUserMedia({ audio: false, video: { ...(deviceId ? { deviceId: { exact: deviceId } } : { facingMode: { ideal: "environment" } }), width: { ideal: resolution.width }, height: { ideal: resolution.height }, @@ -295,7 +302,7 @@ function requestedScaleResolutionDownBy( sender.track && typeof sender.track.getSettings === "function" ? senderCaptureTrack(sender.track).getSettings() : undefined; - const ceiling = QUALITY_RESOLUTIONS[profile.resolution]; + const ceiling = profile.resolution; if (!source?.width || !source.height) { return 1; } diff --git a/src/client/native/wire.ts b/src/client/native/wire.ts index 05b3e39f6..a83e95652 100644 --- a/src/client/native/wire.ts +++ b/src/client/native/wire.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { sfuMediaSchema, maxAudioBitrateSchema } from "../../shared/protocol"; +import { sfuMediaSchema, maxAudioBitrateSchema, qualityResolutionSchema } from "../../shared/protocol"; export const NATIVE_CLIENT_PROTOCOL = 10; export const NATIVE_CLIENT_PORT_START = 39_721; @@ -34,6 +34,7 @@ export const nativeHealthSchema = nativeDiscoveryIdentitySchema.extend({ nativeMedia: z .object({ maxAudioBitrate: maxAudioBitrateSchema.optional(), + maxVideoResolution: qualityResolutionSchema.optional(), receiverReuse: z.boolean().default(false), video: z.boolean().default(false), processAudio: z.boolean().default(false), diff --git a/src/client/pages/ControlsPreview.tsx b/src/client/pages/ControlsPreview.tsx index a2c66c5bc..8f2aca9e4 100644 --- a/src/client/pages/ControlsPreview.tsx +++ b/src/client/pages/ControlsPreview.tsx @@ -19,9 +19,9 @@ import { LauncherForm, type AppMode } from "../components/living/LauncherForm"; import { AppReleaseNotice } from "../components/living/AppReleaseNotice"; import { useTheaterMode } from "../components/living/use-theater-mode"; -import { QualityPresets } from "../components/living/QualityPresets"; +import { QualityPresets, ResolutionControl } from "../components/living/QualityPresets"; import { RoomCodeInput } from "../components/living/RoomCodeInput"; -import type { QualityProfileId } from "../media/quality"; +import { MAX_CAPTURE_RESOLUTION, QUALITY_PROFILES, type QualityProfileId, type QualityResolution } from "../media/quality"; import type { loadCameraPreviews } from "../media/camera-previews"; import { Tooltip } from "../components/living/Tooltip"; import { StatusIndicator } from "../components/living/StatusIndicator"; @@ -57,6 +57,7 @@ export function ControlsPreview() { const [voiceProcessing, setVoiceProcessing] = useState(true); const [microphoneDevice, setMicrophoneDevice] = useState(""); const [sharingSettings, setSharingSettings] = useState(false); + const [resolution, setResolution] = useState(QUALITY_PROFILES["1080p30"].resolution); const [launchMode, setLaunchMode] = useState("link"); const [majorUpdate, setMajorUpdate] = useState(true); const releaseVersion = majorUpdate ? "v2.0.0" : "v1.1.0"; @@ -64,7 +65,7 @@ export function ControlsPreview() { const [launchSite, setLaunchSite] = useState(""); const [launchPassword, setLaunchPassword] = useState(""); const [launchLan, setLaunchLan] = useState("192.0.2.10"); - const [preset, setPreset] = useState("1080p30"); + const [preset, setPreset] = useState("1080p30"); const [policy, setPolicy] = useState<"open" | "private">("open"); const [roomPassword, setRoomPassword] = useState(false); const [includeCredential, setIncludeCredential] = useState(true); @@ -117,7 +118,9 @@ export function ControlsPreview() { } + presets={ { setPreset(id); setResolution(QUALITY_PROFILES[id].resolution); }} />} + picture={ { setResolution(value); setPreset(null); }} />} audio={<>
diff --git a/src/client/pages/HostPage.tsx b/src/client/pages/HostPage.tsx index 5ed8d233e..61886604c 100644 --- a/src/client/pages/HostPage.tsx +++ b/src/client/pages/HostPage.tsx @@ -53,7 +53,7 @@ import type { ComicKind } from "../components/living/Comic"; import type { HintKind } from "../ui/visual-kinds"; import type { ComicTone } from "../components/living/comic-presentation"; import { Tooltip } from "../components/living/Tooltip"; -import { QualityPresets } from "../components/living/QualityPresets"; +import { QualityPresets, ResolutionControl } from "../components/living/QualityPresets"; import { RoomCodeInput, RoomCodeError } from "../components/living/RoomCodeInput"; import { MetricCell } from "../components/living/Metrics"; import { @@ -128,7 +128,8 @@ import { matchingQualityProfileId, QUALITY_PROFILES, DEGRADATION_PREFERENCE_KEYS, - QUALITY_RESOLUTIONS, + MAX_CAPTURE_RESOLUTION, + qualityResolutionEqual, qualitySettingsEqual, qualitySettingsLabel, screenAudioBitrate, @@ -137,7 +138,6 @@ import { setMediaPaused, videoQualitySettingsEqual, type DegradationPreference, - type QualityResolution, type QualitySettings, type ScreenAudioPreset, } from "../media/quality"; @@ -1729,7 +1729,7 @@ export function HostPage({ nextProfile, ); const captureChanged = - previousProfile.resolution !== nextProfile.resolution || + !qualityResolutionEqual(previousProfile.resolution, nextProfile.resolution) || previousProfile.maxFramerate !== nextProfile.maxFramerate; const audioChanged = screenAudioBitrate(previousProfile.maxAudioBitrate) !== @@ -3597,6 +3597,11 @@ export function HostPage({ const audioBitrateCeiling = Math.min(MAX_SCREEN_AUDIO_BITRATE, nativeActive ? nativeClientRef.current?.health.nativeMedia.maxAudioBitrate ?? MAX_SCREEN_AUDIO_BITRATE : MAX_SCREEN_AUDIO_BITRATE); + const nativeResolutionCeiling = nativeActive ? nativeClientRef.current?.health.nativeMedia.maxVideoResolution : undefined; + const resolutionCeiling = { + width: Math.min(MAX_CAPTURE_RESOLUTION.width, nativeResolutionCeiling?.width ?? MAX_CAPTURE_RESOLUTION.width), + height: Math.min(MAX_CAPTURE_RESOLUTION.height, nativeResolutionCeiling?.height ?? MAX_CAPTURE_RESOLUTION.height), + }; return (
@@ -3844,28 +3849,9 @@ export function HostPage({ -
- {( - Object.keys(QUALITY_RESOLUTIONS) as QualityResolution[] - ).map((resolution) => ( - - changeAdvancedQuality({ resolution }) - } - > - {QUALITY_RESOLUTIONS[resolution].label} - - ))} -
+ changeAdvancedQuality({ resolution })} />
diff --git a/src/client/sfu/publisher.ts b/src/client/sfu/publisher.ts index 46514e5f5..04b3893b2 100644 --- a/src/client/sfu/publisher.ts +++ b/src/client/sfu/publisher.ts @@ -8,7 +8,6 @@ import { configureScreenAudioSender, configureVideoSender, needsStartupVideoProfile, - QUALITY_RESOLUTIONS, screenAudioBitrate, screenAudioSenderCeiling, senderParameterWarning, @@ -436,7 +435,7 @@ export class SfuPublisher { private encodings(profile: QualityProfile): RTCRtpEncodingParameters[] { const settings = this.video ? senderCaptureTrack(this.video).getSettings() : undefined; - const ceiling = QUALITY_RESOLUTIONS[profile.resolution]; + const ceiling = profile.resolution; const width = settings?.width ?? ceiling.width; const height = settings?.height ?? ceiling.height; const scale = Math.max(1, width / ceiling.width, height / ceiling.height); @@ -465,7 +464,7 @@ export class SfuPublisher { private media(profile: QualityProfile): SfuMedia { const settings = senderCaptureTrack(this.video!).getSettings(); - const ceiling = QUALITY_RESOLUTIONS[profile.resolution]; + const ceiling = profile.resolution; return { codec: this.codec, layers: this.encodings(profile).map((encoding) => ({ diff --git a/src/client/styles.css b/src/client/styles.css index e93f3ae56..fa225d2ac 100644 --- a/src/client/styles.css +++ b/src/client/styles.css @@ -1178,6 +1178,11 @@ html[data-theme="dark"] .lr-invite-url { background: #1a2634; border-color: #334 .lr-input input[type="password"]:not(:placeholder-shown) { letter-spacing: .25em; } .lr-input.lr-audio-bitrate { width: 160px; max-width: 100%; font-family: var(--mono); } .lr-audio-bitrate span { color: var(--muted); font-size: 12px; } +.lr-input.lr-video-size { display: grid; grid-template-columns: minmax(0, 1fr) auto minmax(0, 1fr) auto; gap: 4px; padding-inline: 8px; width: 252px; max-width: 100%; font-family: var(--mono); } +.lr-resolution-controls { display: flex; flex-direction: column; gap: 8px; min-width: 0; } +.lr-video-size input { width: 100%; padding: 0; text-align: center; appearance: textfield; } +.lr-video-size input::-webkit-inner-spin-button, .lr-video-size input::-webkit-outer-spin-button { appearance: none; margin: 0; } +.lr-video-size span { color: var(--muted); font-size: 12px; } .lr-welcome { width: 100%; min-width: 0; max-width: 100%; margin: 0; overflow: hidden; color: inherit; font-size: 15px; font-weight: 600; line-height: 1.7; text-align: center; white-space: nowrap; text-overflow: ellipsis; } .lr-welcome:not(.is-cipher)::before { content: "✦"; color: #d7982b; margin-right: 8px; } .lr-welcome-cipher { display: inline-flex; align-items: center; gap: 8px; vertical-align: middle; opacity: .72; white-space: nowrap; } diff --git a/src/client/webrtc/video-codec-preflight.ts b/src/client/webrtc/video-codec-preflight.ts index 1084082b7..e7c892027 100644 --- a/src/client/webrtc/video-codec-preflight.ts +++ b/src/client/webrtc/video-codec-preflight.ts @@ -1,6 +1,5 @@ import { configureVideoSender, - QUALITY_RESOLUTIONS, startupVideoProfile, type QualityProfile, } from "../media/quality"; @@ -56,7 +55,7 @@ export function h264ProbeTarget( settings: MediaTrackSettings, profile: QualityProfile, ): H264ProbeTarget { - const ceiling = QUALITY_RESOLUTIONS[profile.resolution]; + const ceiling = profile.resolution; const sourceWidth = positiveNumber(settings.width, ceiling.width); const sourceHeight = positiveNumber(settings.height, ceiling.height); const scale = Math.max( diff --git a/src/shared/protocol.ts b/src/shared/protocol.ts index 87ec7301a..0d5b374bd 100644 --- a/src/shared/protocol.ts +++ b/src/shared/protocol.ts @@ -173,19 +173,12 @@ export const roomCodeSchema = z export const roleSchema = z.enum(["host", "viewer"]); export type Role = z.infer; -export const qualityProfileIdSchema = z.enum([ - "720p30", - "1080p30", - "1080p60", -]); -export type QualityProfileId = z.infer; - -export const qualityResolutionSchema = z.enum([ - "480p", - "720p", - "1080p", - "1440p", -]); +// Describes a requested ceiling, not encoder availability or an allocation. +// Match the existing media-description bound; adapters retain their own limits. +export const qualityResolutionSchema = z.object({ + width: z.number().int().min(4).max(8192).multipleOf(2), + height: z.number().int().min(4).max(8192).multipleOf(2), +}).strict(); export type QualityResolution = z.infer; export const degradationPreferenceSchema = z.enum([ @@ -212,7 +205,7 @@ export const qualitySettingsSchema = z .strict(); export type QualitySettings = z.infer; export const DEFAULT_QUALITY_SETTINGS = { - resolution: "1080p", + resolution: { width: 1920, height: 1080 }, maxFramerate: 30, maxBitrate: 5_000_000, degradationPreference: "balanced", diff --git a/tests/browser-encoding-pool.test.ts b/tests/browser-encoding-pool.test.ts index 16b654428..13209f3c3 100644 --- a/tests/browser-encoding-pool.test.ts +++ b/tests/browser-encoding-pool.test.ts @@ -424,7 +424,7 @@ describe("Browser encoding pool ownership", () => { } if (change === "budget") { strong.demand.budget = 500_000; await sample(); } if (change === "target demand") { weak.demand.budget = 500_000; await sample(); } - if (change === "profile") await strong.handle.updateProfile({ ...profile, resolution: "720p" }); + if (change === "profile") await strong.handle.updateProfile({ ...profile, resolution: { width: 1280, height: 720 } }); const key = ["key dimensions", "quiet current", "unknown current"].includes(change) ? { width: 960, height: 540 } : { width: 1920, height: 1080 }; expect(accept(key)).toBe(false); diff --git a/tests/client-state.test.ts b/tests/client-state.test.ts index 368ed2787..7c9e2b1fa 100644 --- a/tests/client-state.test.ts +++ b/tests/client-state.test.ts @@ -1724,7 +1724,7 @@ describe("client signaling recovery policy", () => { }); const shareGeneration = "share_generation_12345678"; const initialQuality = { - resolution: "1080p", + resolution: { width: 1920, height: 1080 }, maxFramerate: 30, maxBitrate: 5_000_000, degradationPreference: "balanced", @@ -1732,7 +1732,7 @@ describe("client signaling recovery policy", () => { } as const; const firstPending = { ...initialQuality, - resolution: "720p", + resolution: { width: 1280, height: 720 }, maxBitrate: 3_000_000, } as const; const latestPending = { @@ -2041,7 +2041,7 @@ describe("client signaling recovery policy", () => { sfuPublicationGeneration: null, }, qualitySettings: { - resolution: "1080p", + resolution: { width: 1920, height: 1080 }, maxFramerate: 30, maxBitrate: 5_000_000, degradationPreference: "balanced", diff --git a/tests/fixtures/wire-samples.json b/tests/fixtures/wire-samples.json index e08c3df43..62ca26220 100644 --- a/tests/fixtures/wire-samples.json +++ b/tests/fixtures/wire-samples.json @@ -25,7 +25,10 @@ "signalCloseCodeAuthenticationFailed": 4003, "signalCloseCodeViewerAccessRevoked": 4004, "defaultQualitySettings": { - "resolution": "1080p", + "resolution": { + "width": 1920, + "height": 1080 + }, "maxFramerate": 30, "maxBitrate": 5000000, "degradationPreference": "balanced", @@ -55,7 +58,7 @@ { "name": "authenticate-host-full", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"shareGeneration\":\"share_generation_12345678\",\"sharingPaused\":false,\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"viewerPresence\":true,\"displayName\":\"小明\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v24\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"shareGeneration\":\"share_generation_12345678\",\"sharingPaused\":false,\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"viewerPresence\":true,\"displayName\":\"小明\"}", "valid": true }, { @@ -277,37 +280,37 @@ { "name": "set-quality-settings", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", "valid": true }, { "name": "set-quality-settings-with-audio", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000}}", "valid": true }, { "name": "set-quality-settings-legacy-codec", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"videoCodec\":\"vp8\"}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"videoCodec\":\"vp8\"}}", "valid": false }, { "name": "set-quality-settings-framerate-below-floor", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":14,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":14,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", "valid": false }, { "name": "set-quality-settings-fractional-bitrate", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":5000000.5,\"degradationPreference\":\"maintain-resolution\"}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":5000000.5,\"degradationPreference\":\"maintain-resolution\"}}", "valid": false }, { "name": "set-quality-settings-missing-framerate", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", "valid": false }, { @@ -817,31 +820,31 @@ { "name": "set-quality-settings-folded-audio-invalid", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxaudiobitrate\":\"invalid\"}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxaudiobitrate\":\"invalid\"}}", "valid": false }, { "name": "set-quality-settings-folded-audio-null", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxaudiobitrate\":null}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxaudiobitrate\":null}}", "valid": false }, { "name": "set-quality-settings-folded-key-collision", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000,\"maxaudiobitrate\":\"invalid\"}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000,\"maxaudiobitrate\":\"invalid\"}}", "valid": false }, { "name": "set-quality-settings-exact-duplicate-last-valid", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":null,\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":null,\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", "valid": true }, { "name": "set-quality-settings-exact-duplicate-last-null", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxFramerate\":null,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxFramerate\":null,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", "valid": false }, { @@ -1057,61 +1060,61 @@ { "name": "numeric-audio-6000", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":6000}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":6000}}", "valid": true }, { "name": "numeric-audio-96000", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":96000}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":96000}}", "valid": true }, { "name": "numeric-audio-256000", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":256000}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":256000}}", "valid": true }, { "name": "numeric-audio-320000", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":320000}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":320000}}", "valid": true }, { "name": "numeric-audio-510000", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":510000}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":510000}}", "valid": true }, { "name": "numeric-audio-5999", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":5999}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":5999}}", "valid": false }, { "name": "numeric-audio-510001", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":510001}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":510001}}", "valid": false }, { "name": "numeric-audio-128000.5", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000.5}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000.5}}", "valid": false }, { "name": "numeric-audio-null", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":null}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":null}}", "valid": false }, { "name": "numeric-audio-music", "schema": "client", - "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":\"music\"}}", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":\"music\"}}", "valid": false }, { @@ -1119,6 +1122,84 @@ "schema": "client", "json": "{\"type\":\"authenticate\",\"protocol\":\"piik-v23\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", "valid": false + }, + { + "name": "set-quality-settings-numeric-16-10", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1200},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", + "valid": true + }, + { + "name": "set-quality-settings-portrait", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":720,\"height\":1280},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", + "valid": true + }, + { + "name": "set-quality-settings-square", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1080,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", + "valid": true + }, + { + "name": "set-quality-settings-future-4k-description", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":3840,\"height\":2160},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", + "valid": true + }, + { + "name": "set-quality-settings-maximum-description", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":8192,\"height\":8192},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", + "valid": true + }, + { + "name": "set-quality-settings-too-wide", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":8194,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", + "valid": false + }, + { + "name": "set-quality-settings-too-tall", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":8194},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", + "valid": false + }, + { + "name": "set-quality-settings-too-small", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":2,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", + "valid": false + }, + { + "name": "set-quality-settings-odd-size", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1919,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", + "valid": false + }, + { + "name": "set-quality-settings-fractional-size", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080.5},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", + "valid": false + }, + { + "name": "set-quality-settings-missing-height", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", + "valid": false + }, + { + "name": "set-quality-settings-null-size", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":null},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", + "valid": false + }, + { + "name": "set-quality-settings-ui-label-on-wire", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080,\"label\":\"1080p\"},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", + "valid": false } ], "serverMessages": [ @@ -1128,19 +1209,19 @@ }, { "name": "authenticated-host-minimal", - "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v24\",\"peerId\":\"host_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"connectionId\":null,\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"none\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"role\":\"host\",\"viewerPasswordEnabled\":false}" + "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v24\",\"peerId\":\"host_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"connectionId\":null,\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"none\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"role\":\"host\",\"viewerPasswordEnabled\":false}" }, { "name": "authenticated-host-full", - "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v24\",\"peerId\":\"host_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"connectionId\":\"connection_12345678\",\"iceConfig\":{\"iceServers\":[{\"urls\":\"stun:stun.example.test:3478\"},{\"urls\":[\"stun:stun-1.example.test:3478\",\"stun:[2001:db8::1]:3478\"]}],\"natPredictionStunUrls\":[\"stun:nat.example.test:3478\"]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"sfu\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":\"generation_12345678\"},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"role\":\"host\",\"viewerPasswordEnabled\":false}" + "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v24\",\"peerId\":\"host_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"connectionId\":\"connection_12345678\",\"iceConfig\":{\"iceServers\":[{\"urls\":\"stun:stun.example.test:3478\"},{\"urls\":[\"stun:stun-1.example.test:3478\",\"stun:[2001:db8::1]:3478\"]}],\"natPredictionStunUrls\":[\"stun:nat.example.test:3478\"]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"sfu\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":\"generation_12345678\"},\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"role\":\"host\",\"viewerPasswordEnabled\":false}" }, { "name": "authenticated-viewer-minimal", - "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v24\",\"peerId\":\"viewer_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"hostPaused\":false,\"connectionId\":null,\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"none\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"role\":\"viewer\"}" + "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v24\",\"peerId\":\"viewer_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"hostPaused\":false,\"connectionId\":null,\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"none\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"role\":\"viewer\"}" }, { "name": "authenticated-viewer-full", - "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v24\",\"peerId\":\"viewer_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"hostPaused\":true,\"connectionId\":\"connection_12345678\",\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"peer\",\"peerId\":\"parent_12345678\"},\"childPeerIds\":[\"child_12345678\",\"child_87654321\"],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"role\":\"viewer\"}" + "json": "{\"type\":\"authenticated\",\"protocol\":\"piik-v24\",\"peerId\":\"viewer_12345678\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"hostPaused\":true,\"connectionId\":\"connection_12345678\",\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"peer\",\"peerId\":\"parent_12345678\"},\"childPeerIds\":[\"child_12345678\",\"child_87654321\"],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000},\"role\":\"viewer\"}" }, { "name": "signaling-challenge-response", @@ -1196,11 +1277,11 @@ }, { "name": "quality-settings", - "json": "{\"type\":\"quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000}}" + "json": "{\"type\":\"quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"maxAudioBitrate\":128000}}" }, { "name": "quality-settings-without-audio", - "json": "{\"type\":\"quality-settings\",\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}" + "json": "{\"type\":\"quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}" }, { "name": "route-policy", diff --git a/tests/host-peer-tracks.test.ts b/tests/host-peer-tracks.test.ts index 59e812a37..0d367ace6 100644 --- a/tests/host-peer-tracks.test.ts +++ b/tests/host-peer-tracks.test.ts @@ -3377,7 +3377,7 @@ describe("ViewerRelay downstream ownership", () => { it("applies the latest profile to the current and future child", async () => { const customSettings = { - resolution: "1440p", + resolution: { width: 2560, height: 1440 }, maxFramerate: 45, maxBitrate: 10_500_000, degradationPreference: "balanced", diff --git a/tests/host-quality-ownership.test.ts b/tests/host-quality-ownership.test.ts index ba333b917..b800a096a 100644 --- a/tests/host-quality-ownership.test.ts +++ b/tests/host-quality-ownership.test.ts @@ -3,7 +3,7 @@ import { createContext, runInContext } from "node:vm"; import ts from "typescript"; import { describe, expect, it, vi } from "vitest"; import { - QUALITY_PROFILES, qualitySettingsEqual, qualitySettingsLabel, + QUALITY_PROFILES, qualitySettingsEqual, qualitySettingsLabel, qualityResolutionEqual, screenAudioBitrate, videoQualitySettingsEqual, type QualitySettings, } from "../src/client/media/quality"; @@ -108,7 +108,7 @@ function fixture(launchedByClient = true) { setDetails: vi.fn(), setNativeActive: vi.fn(), applyCaptureProfile: vi.fn(async (_stream: unknown, profile: QualitySettings) => { physical = profile; }), captureDetails: () => ({}), - qualitySettingsEqual, videoQualitySettingsEqual, screenAudioBitrate, qualitySettingsLabel, + qualitySettingsEqual, qualityResolutionEqual, videoQualitySettingsEqual, screenAudioBitrate, qualitySettingsLabel, say: (key: string) => key, syncHostSfuQualityWarning: () => null, NativeSenderPeer, discardPreparedHostChild: vi.fn(), removePeer: vi.fn(), startPeer: vi.fn(async () => undefined), nativePreviewTailRef: ref(Promise.resolve()), setSwitchingSource: vi.fn(), diff --git a/tests/native-client.test.ts b/tests/native-client.test.ts index 65dd2071b..ea7038842 100644 --- a/tests/native-client.test.ts +++ b/tests/native-client.test.ts @@ -672,7 +672,7 @@ describe("native App private wire", () => { it("keeps native share lifecycle responses as strict acknowledgements", () => { const profile = { - resolution: "1440p", + resolution: { width: 2560, height: 1440 }, maxFramerate: 60, maxBitrate: 12_000_000, degradationPreference: "maintain-framerate", diff --git a/tests/protocol.test.ts b/tests/protocol.test.ts index c5c7228b9..3cbf4f307 100644 --- a/tests/protocol.test.ts +++ b/tests/protocol.test.ts @@ -44,7 +44,7 @@ it("accepts bounded future reaction events while keeping commands and known prop expect(serverMessageSchema.safeParse({ ...event, sender: { ...event.sender, peerId: "bad" } }).success).toBe(false); }); const qualitySettings = { - resolution: "1080p", + resolution: { width: 1920, height: 1080 }, maxFramerate: 60, maxBitrate: 8_000_000, degradationPreference: "maintain-resolution", @@ -769,7 +769,7 @@ describe("client signaling protocol", () => { expect( clientMessageSchema.safeParse({ type: "set-quality-settings", - qualitySettings: { ...qualitySettings, resolution: "480p" }, + qualitySettings: { ...qualitySettings, resolution: { width: 854, height: 480 } }, }).success, ).toBe(true); for (const maxAudioBitrate of [ @@ -1555,7 +1555,7 @@ describe("server signaling protocol", () => { serverMessageSchema.safeParse({ type: "quality-settings", qualitySettings: { - resolution: "720p", + resolution: { width: 1280, height: 720 }, maxFramerate: 30, maxBitrate: 3_000_000, degradationPreference: "balanced", diff --git a/tests/quality-preference.test.ts b/tests/quality-preference.test.ts index a279e3f09..bc3c51e3e 100644 --- a/tests/quality-preference.test.ts +++ b/tests/quality-preference.test.ts @@ -12,7 +12,7 @@ describe("sharing quality preference", () => { setItem: (key: string, value: string) => storage.set(key, value), } }); expect(readPreferredQuality()).toEqual(DEFAULT_QUALITY_SETTINGS); - const chosen = { ...DEFAULT_QUALITY_SETTINGS, resolution: "1440p" as const, + const chosen = { ...DEFAULT_QUALITY_SETTINGS, resolution: { width: 2560, height: 1440 } as const, maxFramerate: 60, maxBitrate: 10_000_000, degradationPreference: "maintain-resolution" as const, maxAudioBitrate: 256_000 }; savePreferredQuality(chosen); diff --git a/tests/quality.test.ts b/tests/quality.test.ts index d0b435f65..7de190a85 100644 --- a/tests/quality.test.ts +++ b/tests/quality.test.ts @@ -12,6 +12,8 @@ import { QUALITY_PROFILES, QUALITY_RESOLUTIONS, qualitySettingsEqual, + qualityResolutionLabel, + videoQualitySettingsEqual, screenAudioBitrate, screenAudioSenderCeiling, senderParameterWarning, @@ -55,7 +57,7 @@ describe("realtime quality controls", () => { "1080p60", ]); expect(QUALITY_PROFILES["1080p30"]).toMatchObject({ - resolution: "1080p", + resolution: { width: 1920, height: 1080 }, maxFramerate: 30, maxBitrate: 5_000_000, }); @@ -65,11 +67,10 @@ describe("realtime quality controls", () => { expect(QUALITY_RESOLUTIONS["480p"]).toEqual({ width: 854, height: 480, - label: "480p", }); expect( Object.values(QUALITY_PROFILES).map((profile) => profile.resolution), - ).not.toContain("480p"); + ).not.toContainEqual(QUALITY_RESOLUTIONS["480p"]); }); it("bounds initial capture to the selected profile", async () => { @@ -105,19 +106,29 @@ describe("realtime quality controls", () => { const { stream, videoTrack } = createVideoStream(); await applyCaptureProfile(stream, { - resolution: "1440p", + resolution: { width: 1920, height: 1200 }, maxFramerate: 45, maxBitrate: 9_500_000, degradationPreference: "balanced", }); expect(videoTrack.applyConstraints).toHaveBeenCalledWith({ - width: { ideal: 2560, max: 2560 }, - height: { ideal: 1440, max: 1440 }, + width: { ideal: 1920, max: 1920 }, + height: { ideal: 1200, max: 1200 }, frameRate: { ideal: 45, max: 45 }, }); }); + it("compares deserialized size values without treating audio edits as new video", () => { + const profile = QUALITY_PROFILES["1080p30"]; + const restored = JSON.parse(JSON.stringify(profile)); + expect(qualitySettingsEqual(profile, restored)).toBe(true); + expect(videoQualitySettingsEqual(profile, { ...restored, maxAudioBitrate: 256_000 })).toBe(true); + expect(videoQualitySettingsEqual(profile, { ...restored, resolution: { width: 1920, height: 1200 } })).toBe(false); + expect(qualityResolutionLabel(restored.resolution)).toBe("1080p"); + expect(qualityResolutionLabel({ width: 1920, height: 1200 })).toBe("1920×1200"); + }); + it("leaves capture constraints with the owner of a remote track", async () => { const { videoTrack } = createVideoStream(); videoTrack.getCapabilities = vi.fn(() => ({})); From 34d2a3b6aab618d6a0e9536768bb6d0ff64f069d Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:35:36 +0800 Subject: [PATCH 073/262] feat(capture): admit bounded 4K30 video formats --- docs/research/native-client-media.md | 38 +++++++++++++++---- docs/standards/media-quality.md | 10 ++++- docs/status.md | 9 +++-- docs/todo.md | 7 +++- internal/app/app.go | 2 +- .../app/mediaedge/relay_chain_fixture_test.go | 11 +++--- internal/app/mediaedge/relay_derivation.go | 2 +- internal/app/mediaedge/relay_input.go | 13 ++++++- internal/app/mediaedge/relay_input_test.go | 8 ++-- internal/app/nativecapture/protocol.go | 26 +++++++++++-- internal/app/nativecapture/protocol_test.go | 12 +++++- internal/app/nativecapture/stream.go | 7 ++-- internal/app/nativecontrol/session_test.go | 8 +++- native/capture/darwin/main.swift | 26 ++++++++----- native/capture/linux/main.c | 22 ++++++++--- native/capture/linux/main.test.c | 12 +++++- native/capture/windows/capture_geometry.h | 7 +--- native/capture/windows/capture_input.h | 4 +- native/capture/windows/h264_decoder.h | 14 +++---- native/capture/windows/main.cpp | 11 +++--- native/capture/windows/output_worker.test.cpp | 14 ++++++- native/capture/windows/video_bounds.h | 30 +++++++++++++++ native/capture/windows/vp8_decoder.h | 6 ++- 23 files changed, 223 insertions(+), 76 deletions(-) create mode 100644 native/capture/windows/video_bounds.h diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 1e8c965b6..e2d428e9e 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -567,15 +567,13 @@ padding. Physical game, macOS/Linux and orientation-change acceptance remain ope The candidate's Native capture admission now shares the numeric dimension bounds already used by encoded outputs instead of duplicating the four UI -presets. Even dimensions must fit within 2560x1440; a source ceiling must also -yield a valid half-size output. Frame-rate, bitrate and output-count bounds -remain unchanged. Candidate signaling and Native control now carry numeric +presets. A source ceiling must also yield a valid half-size output. Bitrate and +output-count bounds remain unchanged. Candidate signaling and Native control now carry numeric width/height ceilings, with custom input beside the existing UI shortcuts. Descriptive wire bounds are independent of the current implementation maximum; -App discovery advertises that maximum, and Native rejects unsupported profiles -as operation failures without retiring the control session. This does not yet -admit product 4K. Removing an enum is not permission to allocate arbitrary media -resources. +App discovery advertises per-axis maxima, and Native rejects unsupported +size/rate combinations as operation failures without retiring the control +session. Removing an enum is not permission to allocate arbitrary media resources. The Windows H.264 UI gate applies a non-preset 960x600 ceiling during a live share and observes fitted output without replacing the Viewer media object. @@ -584,6 +582,32 @@ sharing still pass. Value-based comparison prevents an identical deserialized size from turning an audio-only edit into a video update. Numeric preference round-trips and malformed/unsupported requests have separate boundary checks. +Native bounds now admit 3840x2160 and 2160x3840 at 30 fps. The per-axis limit and +pixel total remain independent; an arbitrary 3840-square picture is rejected. +The output throughput ceiling uses H.264 Level 5.1's macroblock rate, including +alignment for non-preset sizes. Coded padding and decoded reference buffers have +their own pre-allocation bounds, following [H.264 Table A-1](https://www.itu.int/rec/T-REC-H.264) +(also implemented in [FFmpeg's level table](https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/h264_levels.c)). +The Windows decoder's advertised support remains +[Level 5.1](https://learn.microsoft.com/en-us/windows/win32/medfound/h-264-video-decoder). +An isolated Level 5.2 trial was rejected by this machine's hardware encoder at +output-type setup; it did not reach the decoder. The candidate therefore keeps +Level 5.1 and rejects 4K60 before allocating an encoder. + +Current production Windows code encoded and decoded 30 synthetic frames in each +4K orientation, both with and without a D3D decoder manager. Retaining/uploading +the decoded surface also passed. The normal Go/native encoded-input path then +produced 1920x1080 and 1080x1920 derived outputs. Production H.264 and VP8 output +workers also handled a 4K-to-1080p-to-4K synthetic source sequence, and Native +VP8 derivation produced the lower output from the 4K input. The Browser H.264 SFU gate with +a synthetic 3840x2160/30 source passed first-picture delivery, an explicit lower +profile, audio-only edits with retained connections, subscription/page recovery +and retirement. Windows native checks and Linux headless build/checks pass; +macOS SDK and physical 4K capture are not verified here. The UI ceiling remains +2560x1440 until combined capability controls and whole-route acceptance are +complete. These checks do not establish 4K60, sustained device performance or +universal portrait decoding support. + The proposed boundary follows the [Media Capture specification](https://www.w3.org/TR/mediacapture-streams/#constrainable-interface): requested constraints, supported capabilities and actual settings have different diff --git a/docs/standards/media-quality.md b/docs/standards/media-quality.md index 8e713f29d..d5857cdeb 100644 --- a/docs/standards/media-quality.md +++ b/docs/standards/media-quality.md @@ -128,7 +128,7 @@ The three recommended profiles are ceilings, not delivery guarantees: `1080p30` is the default. The 2.0 candidate carries numeric width/height ceilings; 480p, 720p, 1080p and 1440p remain UI shortcuts. Advanced picture settings also -accept custom even dimensions within the current 2560x1440 implementation bound. +accept custom even dimensions within the current 2560x1440 UI bound. Width and height form one applied edit. Larger wire-valid requests do not authorize an adapter to exceed its local capture, codec or allocation limits. Frame rate, bitrate, and `maintain-resolution | balanced | @@ -145,6 +145,14 @@ derivation, validate every H.264 sequence parameter set's supported format, coded storage and visible crop; a later small descriptor cannot authorize earlier large input. +The candidate Native adapters admit even visible dimensions up to 3840 per +axis and 3840x2160 total pixels. Their shared output-rate bound follows H.264 +Level 5.1: 983,040 16x16 macroblocks per second, with at most 60 fps. This admits +landscape/portrait 4K30 and existing 1440p60, not 4K60. Decoder alignment has a +separate coded-storage bound; reference-picture storage is bounded before +Native derivation. These are admission limits, not a device performance promise. +The UI remains within its previous ceiling pending whole-route acceptance. + Framework-driven downscaling under bandwidth or device pressure is valid within the selected degradation preference. A low decoded resolution alone is not a defect and does not justify a new quality floor. Review avoidable disruption, diff --git a/docs/status.md b/docs/status.md index 6039e2cf8..8fcc633a7 100644 --- a/docs/status.md +++ b/docs/status.md @@ -67,9 +67,12 @@ capture v7 and SQLite schema 2 remain unchanged. Matched candidate App/Web/Server builds are required; 1.x participants are rejected by the existing compatibility checks. Custom width/height controls retain the current implementation ceiling. -Windows and Linux output now preserve source shape within that ceiling; -axis-independent limits, Browser/macOS geometry, 4K and additional codecs are -still assessed through the media boundary. +Windows and Linux output now preserve source shape within that ceiling. +Native admission now has axis-independent size, pixel-count and output-rate +bounds for 4K30; local Windows encoding, decoding and derived-output checks pass. +Browser H.264 4K30 also passes the existing SFU flow with a synthetic source. +Browser/macOS geometry, 4K controls and whole-route/device acceptance, and +additional codecs remain in the media assessment. Native Magicsock remains an isolated experiment with incomplete product integration and unresolved recovery cases. These proposals do not yet change the accepted contract or transport. A small external test is proposed before diff --git a/docs/todo.md b/docs/todo.md index b5f27e894..ba3d02fc0 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -54,8 +54,11 @@ Implementation needs a concrete useful result within the product model. capture gate within current implementation limits. Next assess source-aspect-preserving geometry and 4K through the same owners. Windows and Linux outputs now fit the source without a fixed padded landscape canvas; - axis-independent limits, Browser/macOS geometry and product 4K - admission remain open. Preserve + Native axis-independent 4K30 admission and bounded Windows codec/derived-output + checks now pass, as does Browser H.264 4K30 over the existing SFU gate with a + synthetic source. Next align controls with combined size/rate limits and verify + Native/Browser mixed routes, source/live changes and limited receivers. + Browser/macOS geometry and physical 4K capture remain open. Preserve safety limits and framework-owned adaptation. The [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment) retain explicit receive-limit enforcement and the current 320 kbps sender diff --git a/internal/app/app.go b/internal/app/app.go index 2e38d3430..b19955222 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -629,7 +629,7 @@ func discoverNativeMedia(ctx context.Context, configuredPath string) nativeRunti capture: capabilities, capabilities: loopback.NativeMediaCapabilities{ MaxVideoResolution: protocol.QualityResolution{ - Width: nativecapture.MaxVideoWidth, Height: nativecapture.MaxVideoHeight, + Width: nativecapture.MaxVideoDimension, Height: nativecapture.MaxVideoDimension, }, MaxAudioBitrate: nativeaudio.MaxSendBitrate, ReceiverReuse: true, diff --git a/internal/app/mediaedge/relay_chain_fixture_test.go b/internal/app/mediaedge/relay_chain_fixture_test.go index 7ad132931..5d1170a8e 100644 --- a/internal/app/mediaedge/relay_chain_fixture_test.go +++ b/internal/app/mediaedge/relay_chain_fixture_test.go @@ -44,10 +44,11 @@ func TestRelayChainEncodedFixture(t *testing.T) { t.Fatal("encoded chain fixture exceeds 64 MiB") } type fixtureFrame struct { - Index, Width, Height int - Recovery bool - DataHex string - Data []byte `json:"-"` + Index int + Width, Height uint32 + Recovery bool + DataHex string + Data []byte `json:"-"` } var frames []fixtureFrame identities := make(map[[sha256.Size]byte][]int) @@ -59,7 +60,7 @@ func TestRelayChainEncodedFixture(t *testing.T) { } check(err) if len(frames) >= 400 || frame.Index != len(frames) || frame.Width <= 0 || frame.Height <= 0 || - frame.Width > 2560 || frame.Height > 1440 || len(frame.DataHex) == 0 || len(frame.DataHex) > 8<<20 || + !nativecapture.ValidVideoSize(frame.Width, frame.Height) || len(frame.DataHex) == 0 || len(frame.DataHex) > 8<<20 || len(frame.DataHex)%2 != 0 { t.Fatal("encoded chain fixture has invalid or excessive frames") } diff --git a/internal/app/mediaedge/relay_derivation.go b/internal/app/mediaedge/relay_derivation.go index 821b68dcc..1969caf65 100644 --- a/internal/app/mediaedge/relay_derivation.go +++ b/internal/app/mediaedge/relay_derivation.go @@ -132,7 +132,7 @@ func (source *Source) relayPlan() relayPlan { plan.output = nativecapture.ScreenShareOutputs(*plan.profile)[0] plan.output.Width = min(width/4*2, plan.output.Width) plan.output.Height = min(height/4*2, plan.output.Height) - if !plan.output.Valid() || width > 2560 || height > 1440 { + if !plan.output.Valid() || !nativecapture.ValidVideoSize(width, height) { return plan } demands := source.collectDemands(edges, nil) diff --git a/internal/app/mediaedge/relay_input.go b/internal/app/mediaedge/relay_input.go index c0e78b4fe..a1c98dfe5 100644 --- a/internal/app/mediaedge/relay_input.go +++ b/internal/app/mediaedge/relay_input.go @@ -151,10 +151,19 @@ func relayH264Size(nal []byte) (uint32, uint32, error) { } // Constrained Baseline uses 8-bit 4:2:0. Bound uncropped storage before // allowing padding to be removed from the visible picture. - if sps.ChromaFormatIdc != 1 || sps.MaxNumRefFrames > 16 || - width > nativecapture.MaxVideoWidth || height > nativecapture.MaxVideoHeight { + if sps.ChromaFormatIdc != 1 || !nativecapture.ValidCodedVideoSize(width, height) { return 0, 0, errors.New("native H264 input storage exceeds its bound") } + bufferedFrames := max(1, sps.MaxNumRefFrames) + if sps.VUI != nil && sps.VUI.BitstreamRestriction != nil { + bufferedFrames = max(bufferedFrames, sps.VUI.BitstreamRestriction.MaxDecFrameBuffering, + sps.VUI.BitstreamRestriction.MaxNumReorderFrames) + } + // Level 5.1 also bounds the decoded-picture buffer. A per-frame size check + // alone would let a larger picture multiply the old 16-reference allocation. + if bufferedFrames > 16 || width*height/256*uint64(bufferedFrames) > 184_320 { + return 0, 0, errors.New("native H264 input reference storage exceeds its bound") + } if crop := sps.FrameCropping; crop != nil { x := (uint64(crop.LeftOffset) + uint64(crop.RightOffset)) * 2 y := (uint64(crop.TopOffset) + uint64(crop.BottomOffset)) * cropUnitY diff --git a/internal/app/mediaedge/relay_input_test.go b/internal/app/mediaedge/relay_input_test.go index 3ba2e0dea..f44f397ea 100644 --- a/internal/app/mediaedge/relay_input_test.go +++ b/internal/app/mediaedge/relay_input_test.go @@ -106,9 +106,11 @@ func TestRelayH264AllocationBounds(t *testing.T) { {"1440p", "6742c033da00a002d640", 2560, 1440}, {"padded-1080p", "6742c033da01e0089f95", 1920, 1080}, {"portrait-within-bound", "6742c033da01100b5e5d", 1080, 1440}, - {"4k", "6742c033da00f0010f90", 0, 0}, - {"portrait-overflow", "6742c033da008701e190", 0, 0}, - {"padded-4k", "6742c033da00f00111f894", 0, 0}, + {"4k", "6742c033da00f0010f90", 3840, 2160}, + {"portrait-4k", "6742c033da008701e190", 2160, 3840}, + {"padded-4k", "6742c033da00f00111f894", 3840, 2160}, + {"4k-reference-buffer-limit", "6742c033f300780087c8", 3840, 2160}, + {"4k-reference-buffer-overflow", "6742c033f380780087c8", 0, 0}, {"large-cropped-storage", "6742c033da00f001e1f8034940", 0, 0}, {"crop-overflow", "6742c033da0a37e000000300100000030004", 0, 0}, {"coded-size-overflow", "6742c033da000003000040000003000d90", 0, 0}, diff --git a/internal/app/nativecapture/protocol.go b/internal/app/nativecapture/protocol.go index 5ba4f96a4..236e38eff 100644 --- a/internal/app/nativecapture/protocol.go +++ b/internal/app/nativecapture/protocol.go @@ -15,14 +15,32 @@ const ( maxMediaBytes = encoded.MaxAccessUnitBytes maxStatusBytes = 4 * 1024 maxControlBytes = 64 - MaxVideoWidth = 2560 - MaxVideoHeight = 1440 + MaxVideoDimension = 3840 + MaxVideoPixels = 3840 * 2160 + // Coded pictures may include alignment outside the visible crop. Keep this + // separate from visible dimensions and within H.264 Level 5.1 storage. + MaxCodedVideoDimension = 4096 + MaxCodedVideoPixels = 36_864 * 16 * 16 ) // ValidVideoSize bounds the decoded picture in the current capture contract. func ValidVideoSize(width, height uint32) bool { - return width >= 2 && width <= MaxVideoWidth && width%2 == 0 && - height >= 2 && height <= MaxVideoHeight && height%2 == 0 + return width >= 2 && width <= MaxVideoDimension && width%2 == 0 && + height >= 2 && height <= MaxVideoDimension && height%2 == 0 && + uint64(width)*uint64(height) <= MaxVideoPixels +} + +// ValidVideoFormat keeps both current encoders within the shared receive +// ceiling (H.264 Level 5.1), including non-preset shapes and frame rates. +func ValidVideoFormat(width, height, framerate uint32) bool { + return ValidVideoSize(width, height) && framerate >= 1 && framerate <= 60 && + uint64((width+15)/16)*uint64((height+15)/16)*uint64(framerate) <= 983_040 +} + +func ValidCodedVideoSize(width, height uint64) bool { + return width >= 2 && width <= MaxCodedVideoDimension && width%2 == 0 && + height >= 2 && height <= MaxCodedVideoDimension && height%2 == 0 && + width*height <= MaxCodedVideoPixels } type FrameKind uint8 diff --git a/internal/app/nativecapture/protocol_test.go b/internal/app/nativecapture/protocol_test.go index b356ada63..8ef21f0be 100644 --- a/internal/app/nativecapture/protocol_test.go +++ b/internal/app/nativecapture/protocol_test.go @@ -292,6 +292,9 @@ func TestVideoProfileUsesTheProductBounds(t *testing.T) { {Width: 2560, Height: 1440, Framerate: 60, Bitrate: 12_000_000, Preference: "balanced"}, {Width: 1920, Height: 1200, Framerate: 30, Bitrate: 5_000_000, Preference: "balanced"}, {Width: 720, Height: 1280, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 3840, Height: 2160, Framerate: 30, Bitrate: 12_000_000, Preference: "balanced"}, + {Width: 2160, Height: 3840, Framerate: 30, Bitrate: 12_000_000, Preference: "balanced"}, + {Width: 3000, Height: 2764, Framerate: 30, Bitrate: 12_000_000, Preference: "balanced"}, {Width: 4, Height: 4, Framerate: 15, Bitrate: 2_000_000, Preference: "balanced"}, } { if !profile.Valid() { @@ -309,8 +312,13 @@ func TestVideoProfileUsesTheProductBounds(t *testing.T) { {Width: 1280, Height: 2, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, {Width: 1279, Height: 720, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, {Width: 1280, Height: 719, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, - {Width: 2562, Height: 1440, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, - {Width: 2560, Height: 1442, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 3842, Height: 2160, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 2160, Height: 3842, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 3840, Height: 2162, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 3000, Height: 2766, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 3840, Height: 2160, Framerate: 31, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 2160, Height: 3840, Framerate: 60, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: ^uint32(0), Height: ^uint32(0), Framerate: ^uint32(0), Bitrate: 3_000_000, Preference: "balanced"}, {Width: 1920, Height: 1080, Framerate: 14, Bitrate: 5_000_000, Preference: "balanced"}, {Width: 1920, Height: 1080, Framerate: 30, Bitrate: 12_000_001, Preference: "balanced"}, {Width: 1920, Height: 1080, Framerate: 30, Bitrate: 5_000_000, Preference: "unknown"}, diff --git a/internal/app/nativecapture/stream.go b/internal/app/nativecapture/stream.go index 205eb63d9..f8626c602 100644 --- a/internal/app/nativecapture/stream.go +++ b/internal/app/nativecapture/stream.go @@ -88,8 +88,7 @@ type OutputProfile struct { } func (profile OutputProfile) Valid() bool { - return ValidVideoSize(profile.Width, profile.Height) && - profile.Framerate >= 1 && profile.Framerate <= 60 && + return ValidVideoFormat(profile.Width, profile.Height, profile.Framerate) && profile.Bitrate >= minOutputBitrate && profile.Bitrate <= maxOutputBitrate } @@ -111,8 +110,8 @@ func ScreenShareOutputs(profile VideoProfile) []OutputProfile { func (profile VideoProfile) Valid() bool { // The source ceiling must also yield a valid half-size output. Preset labels // belong to the UI, not the capture process's numeric resource boundary. - return ValidVideoSize(profile.Width, profile.Height) && profile.Width >= 4 && profile.Height >= 4 && - profile.Framerate >= 15 && profile.Framerate <= 60 && + return ValidVideoFormat(profile.Width, profile.Height, profile.Framerate) && profile.Width >= 4 && profile.Height >= 4 && + profile.Framerate >= 15 && profile.Bitrate >= 2_000_000 && profile.Bitrate <= 12_000_000 && validVideoPreference(profile.Preference) } diff --git a/internal/app/nativecontrol/session_test.go b/internal/app/nativecontrol/session_test.go index 09e240589..4e3f633a7 100644 --- a/internal/app/nativecontrol/session_test.go +++ b/internal/app/nativecontrol/session_test.go @@ -671,7 +671,11 @@ func TestQualitySettingsMapOnceIntoNativeMedia(t *testing.T) { } settings.Resolution = protocol.QualityResolution{Width: 3840, Height: 2160} if nativeQualityProfile(settings).Valid() { - t.Fatal("unsupported resolution was accepted") + t.Fatal("unsupported 4K60 combination was accepted") + } + settings.MaxFramerate = 30 + if !nativeQualityProfile(settings).Valid() { + t.Fatal("4K30 was rejected") } } @@ -685,7 +689,7 @@ func TestUpdateShareRequiresTheCurrentStrictProfile(t *testing.T) { assertOperationFailure(t, session, strings.Replace(valid, `"maxAudioBitrate":128000`, `"maxAudioBitrate":510000`, 1)) assertOperationFailure(t, session, strings.Replace(valid, `"resolution":{"width":1920,"height":1080}`, - `"resolution":{"width":3840,"height":2160}`, 1)) + `"resolution":{"width":7680,"height":4320}`, 1)) if _, err := session.Handle(t.Context(), []byte(`{"version":10,"id":"request_options","type":"capture-options"}`)); err != nil { t.Fatalf("unsupported media setting retired control: %v", err) } diff --git a/native/capture/darwin/main.swift b/native/capture/darwin/main.swift index 05a72adbb..6e05e6a3a 100644 --- a/native/capture/darwin/main.swift +++ b/native/capture/darwin/main.swift @@ -15,6 +15,8 @@ private let height = 720 private let frameRate: Int32 = 30 private let bitrate = 3_000_000 private let maxPayloadBytes = 4_194_304 +private let maxVideoDimension = 3840 +private let maxVideoPixels = 3840 * 2160 private let supportedH264Levels: Set = [ 0x1e, 0x1f, 0x20, 0x28, 0x29, 0x2a, 0x32, 0x33, ] @@ -1341,8 +1343,15 @@ private func captureFilter( } private func validVideoSize(_ width: Int, _ height: Int) -> Bool { - width >= 2 && width <= 2560 && width % 2 == 0 && - height >= 2 && height <= 1440 && height % 2 == 0 + width >= 2 && width <= maxVideoDimension && width % 2 == 0 && + height >= 2 && height <= maxVideoDimension && height % 2 == 0 && + width * height <= maxVideoPixels +} + +private func validVideoFormat(_ width: Int, _ height: Int, _ fps: Int32) -> Bool { + // Match the shared H.264 Level 5.1 receive ceiling, including numeric sizes. + validVideoSize(width, height) && fps >= 1 && fps <= 60 && + ((width + 15) / 16) * ((height + 15) / 16) * Int(fps) <= 983_040 } private func videoProfile(_ arguments: [String]) throws -> VideoProfile { @@ -1358,8 +1367,8 @@ private func videoProfile(_ arguments: [String]) throws -> VideoProfile { arguments[22] == "--protocol-v7" else { throw CaptureFailure(description: "invalid video profile arguments") } - guard validVideoSize(width, height), width >= 4, height >= 4, - frameRate >= 15, frameRate <= 60, + guard validVideoFormat(width, height, frameRate), width >= 4, height >= 4, + frameRate >= 15, bitrate >= 2_000_000, bitrate <= 12_000_000 else { throw CaptureFailure(description: "video profile is outside the product bounds") } @@ -1385,9 +1394,9 @@ private func outputProfiles(_ arguments: [String], start: Int = 23, let height = Int(arguments[index + 2]), let fps = Int32(arguments[index + 3]), let bitrate = Int(arguments[index + 4]), - validVideoSize(width, height), - width <= (source?.width ?? 2560), height <= (source?.height ?? 1440), - fps >= 1, fps <= (source?.frameRate ?? 60), + validVideoFormat(width, height, fps), + width <= (source?.width ?? maxVideoDimension), height <= (source?.height ?? maxVideoDimension), + fps <= (source?.frameRate ?? 60), bitrate >= 1000, bitrate <= (source?.bitrate ?? 12_000_000) else { throw CaptureFailure(description: "output profile is outside source bounds") } @@ -1569,8 +1578,7 @@ private final class H264InputDecoder { try require(status, "decode-h264-format") guard let nextFormat else { throw CaptureFailure(description: "missing input format") } let dimensions = CMVideoFormatDescriptionGetDimensions(nextFormat) - guard dimensions.width >= 2, dimensions.width <= 2560, - dimensions.height >= 2, dimensions.height <= 1440 else { + guard validVideoSize(Int(dimensions.width), Int(dimensions.height)) else { throw CaptureFailure(description: "decoded input exceeds its dimension bound") } if let session, !VTDecompressionSessionCanAcceptFormatDescription(session, formatDescription: nextFormat) { diff --git a/native/capture/linux/main.c b/native/capture/linux/main.c index a0baf05fa..575cc5443 100644 --- a/native/capture/linux/main.c +++ b/native/capture/linux/main.c @@ -19,6 +19,8 @@ enum { kMaxOutputs = 6, kMaxPayloadBytes = 4 * 1024 * 1024, kAudioFrameBytes = 960 * 2 * 2, + kMaxVideoDimension = 3840, + kMaxVideoPixels = 3840 * 2160, }; typedef enum { @@ -371,8 +373,15 @@ static gboolean parse_uint(const char *value, guint minimum, guint maximum, } static gboolean valid_video_size(guint width, guint height) { - return width >= 2 && width <= 2560 && width % 2 == 0 && - height >= 2 && height <= 1440 && height % 2 == 0; + return width >= 2 && width <= kMaxVideoDimension && width % 2 == 0 && + height >= 2 && height <= kMaxVideoDimension && height % 2 == 0 && + (guint64)width * height <= kMaxVideoPixels; +} + +static gboolean valid_video_format(guint width, guint height, guint fps) { + // Match the shared H.264 Level 5.1 receive ceiling, including numeric sizes. + return valid_video_size(width, height) && fps >= 1 && fps <= 60 && + (guint64)((width + 15) / 16) * ((height + 15) / 16) * fps <= 983040; } static gboolean parse_outputs(int count, char **values, guint start, CaptureRun *run) { @@ -390,11 +399,12 @@ static gboolean parse_outputs(int count, char **values, guint start, CaptureRun output->activation_timestamp = GST_CLOCK_TIME_NONE; output->profile.preference = run->profile.preference; if (strcmp(values[offset], "--output") != 0 || - !parse_uint(values[offset + 1], 2, 2560, &output->profile.width) || - !parse_uint(values[offset + 2], 2, 1440, &output->profile.height) || + !parse_uint(values[offset + 1], 2, kMaxVideoDimension, &output->profile.width) || + !parse_uint(values[offset + 2], 2, kMaxVideoDimension, &output->profile.height) || !parse_uint(values[offset + 3], 1, 60, &output->profile.frame_rate) || !parse_uint(values[offset + 4], 1000, 12000000, &output->profile.bit_rate) || - !valid_video_size(output->profile.width, output->profile.height)) return FALSE; + !valid_video_format(output->profile.width, output->profile.height, + output->profile.frame_rate)) return FALSE; } return TRUE; } @@ -448,7 +458,7 @@ static gboolean parse_profile(int count, char **values, CaptureRun *run, !parse_uint(values[17], 2000000, 12000000, &profile->bit_rate)) { return FALSE; } - if (!valid_video_size(profile->width, profile->height) || + if (!valid_video_format(profile->width, profile->height, profile->frame_rate) || profile->width < 4 || profile->height < 4) return FALSE; if (!parse_preference(values[19], profile) || !parse_outputs(count, values, 23, run)) return FALSE; run->original_output = MIN(1, run->output_count - 1); diff --git a/native/capture/linux/main.test.c b/native/capture/linux/main.test.c index 33a60e967..e2f3548ac 100644 --- a/native/capture/linux/main.test.c +++ b/native/capture/linux/main.test.c @@ -162,15 +162,25 @@ static void check_slot_profiles(void) { "--output", "1920", "1200", "30", "3000000"}; const struct { char *width; char *height; gboolean valid; } sizes[] = { {"1920", "1200", TRUE}, {"720", "1280", TRUE}, {"4", "4", TRUE}, + {"3840", "2160", TRUE}, {"2160", "3840", TRUE}, {"3000", "2764", TRUE}, {"0", "720", FALSE}, {"2", "720", FALSE}, {"1280", "2", FALSE}, {"1279", "720", FALSE}, {"1280", "719", FALSE}, - {"2562", "1440", FALSE}, {"2560", "1442", FALSE}, {"3840", "2160", FALSE}}; + {"3842", "2160", FALSE}, {"2160", "3842", FALSE}, + {"3840", "2162", FALSE}, {"3000", "2766", FALSE}}; for (guint index = 0; index < G_N_ELEMENTS(sizes); ++index) { numeric[11] = numeric[24] = sizes[index].width; numeric[13] = numeric[25] = sizes[index].height; run = (CaptureRun){0}; g_assert_cmpint(parse_profile(G_N_ELEMENTS(numeric), numeric, &run, &encoder), ==, sizes[index].valid); } + numeric[11] = numeric[24] = "3840"; + numeric[13] = numeric[25] = "2160"; + numeric[15] = numeric[26] = "60"; + run = (CaptureRun){0}; + g_assert_false(parse_profile(G_N_ELEMENTS(numeric), numeric, &run, &encoder)); + numeric[15] = "30"; // Reject an excessive derived output independently. + g_assert_false(parse_profile(G_N_ELEMENTS(numeric), numeric, &run, &encoder)); + g_assert_false(valid_video_format(G_MAXUINT, G_MAXUINT, G_MAXUINT)); } static GstPadProbeReturn record_key_request(GstPad *pad, GstPadProbeInfo *info, diff --git a/native/capture/windows/capture_geometry.h b/native/capture/windows/capture_geometry.h index d73729873..5e199890c 100644 --- a/native/capture/windows/capture_geometry.h +++ b/native/capture/windows/capture_geometry.h @@ -1,5 +1,7 @@ #pragma once +#include "video_bounds.h" + #include #include @@ -8,11 +10,6 @@ namespace piik::capture { -inline bool ValidVideoSize(UINT32 width, UINT32 height) { - return width >= 2 && width <= 2560 && !(width & 1) && - height >= 2 && height <= 1440 && !(height & 1); -} - inline SIZE DisplayedFrameSize(SIZE captured, SIZE desktop, SIZE target, DISPLAYCONFIG_SCALING scaling, DISPLAYCONFIG_ROTATION rotation) { diff --git a/native/capture/windows/capture_input.h b/native/capture/windows/capture_input.h index 8d56f29dc..1dee4fbe0 100644 --- a/native/capture/windows/capture_input.h +++ b/native/capture/windows/capture_input.h @@ -1,6 +1,7 @@ #pragma once #include "capture_control.h" +#include "video_bounds.h" #include #include @@ -85,8 +86,7 @@ class InputEnvelopes final { } else if ((frame_.kind != 2 && frame_.kind != 4) || header_[6] > 1 || frame_.duration == 0 || frame_.timestamp > static_cast(INT64_MAX) / 100 || frame_.duration > static_cast(INT64_MAX) / 100 || - frame_.width < 2 || frame_.height < 2 || frame_.width > 2560 || frame_.height > 1440 || - (frame_.width & 1) || (frame_.height & 1) || payload_size_ > 4 * 1024 * 1024) { + !ValidVideoSize(frame_.width, frame_.height) || payload_size_ > 4 * 1024 * 1024) { throw std::runtime_error("Invalid native encoded input"); } frame_.data.reserve(payload_size_); diff --git a/native/capture/windows/h264_decoder.h b/native/capture/windows/h264_decoder.h index f7e0bd55f..af776bc95 100644 --- a/native/capture/windows/h264_decoder.h +++ b/native/capture/windows/h264_decoder.h @@ -41,8 +41,8 @@ class H264Decoder final { Check(decoder_.As(&codec), "codec-api"); for (const auto& setting : { std::pair{CODECAPI_AVLowLatencyMode, TRUE}, - std::pair{CODECAPI_AVDecVideoMaxCodedWidth, 2560}, - std::pair{CODECAPI_AVDecVideoMaxCodedHeight, 1440}}) { + std::pair{CODECAPI_AVDecVideoMaxCodedWidth, kMaxCodedVideoDimension}, + std::pair{CODECAPI_AVDecVideoMaxCodedHeight, kMaxCodedVideoDimension}}) { VARIANT value; VariantInit(&value); value.vt = VT_UI4; @@ -117,7 +117,7 @@ class H264Decoder final { if (subtype != MFVideoFormat_NV12) continue; UINT32 width = 0, height = 0; Check(MFGetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, &width, &height), "output-size"); - if (width < 2 || height < 2 || width > 2560 || height > 1440 || (width & 1) || (height & 1)) { + if (!ValidVideoSize(width, height)) { throw std::runtime_error("H264 decoder output dimensions exceed the source bound"); } Check(decoder_->SetOutputType(0, type.Get(), 0), "set-output"); @@ -142,11 +142,11 @@ class H264Decoder final { Check(decoder_->GetOutputStreamInfo(0, &info), "output-info"); Microsoft::WRL::ComPtr owned; if (!(info.dwFlags & (MFT_OUTPUT_STREAM_PROVIDES_SAMPLES | MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES))) { - const DWORD expected = static_cast(stride_) * height_ * 3 / 2; - const DWORD size = std::max(info.cbSize, expected); - if (size > 2560 * 1440 * 2) throw std::runtime_error("H264 decoder output allocation exceeds its bound"); + const UINT64 expected = static_cast(stride_) * height_ * 3 / 2; + const UINT64 size = std::max(info.cbSize, expected); + if (size > kMaxCodedVideoPixels * 2) throw std::runtime_error("H264 decoder output allocation exceeds its bound"); Microsoft::WRL::ComPtr buffer; - Check(MFCreateAlignedMemoryBuffer(size, info.cbAlignment ? info.cbAlignment - 1 : 0, &buffer), "output-buffer"); + Check(MFCreateAlignedMemoryBuffer(static_cast(size), info.cbAlignment ? info.cbAlignment - 1 : 0, &buffer), "output-buffer"); Check(MFCreateSample(&owned), "output-sample"); Check(owned->AddBuffer(buffer.Get()), "output-sample-buffer"); } diff --git a/native/capture/windows/main.cpp b/native/capture/windows/main.cpp index b80b1261e..5ff8a09d1 100644 --- a/native/capture/windows/main.cpp +++ b/native/capture/windows/main.cpp @@ -966,8 +966,8 @@ DegradationPreference ParseDegradationPreference(const wchar_t* value) { } void ValidateVideoProfile(const VideoProfile& profile) { - if (!piik::capture::ValidVideoSize(profile.width, profile.height) || - profile.width < 4 || profile.height < 4 || profile.frame_rate < 15 || profile.frame_rate > 60 || + if (!piik::capture::ValidVideoFormat(profile.width, profile.height, profile.frame_rate) || + profile.width < 4 || profile.height < 4 || profile.frame_rate < 15 || profile.bit_rate < 2'000'000 || profile.bit_rate > 12'000'000) { Fail("argument-profile", "video profile is outside the product bounds"); } @@ -981,8 +981,8 @@ void ParseOutputProfiles(ProductArguments& arguments, int first, int count, wcha output.height = ParseIndex(values[index + 2], "argument-output-height"); output.frame_rate = ParseIndex(values[index + 3], "argument-output-fps"); output.bit_rate = ParseIndex(values[index + 4], "argument-output-bitrate"); - if (!piik::capture::ValidVideoSize(output.width, output.height) || output.frame_rate == 0 || - output.frame_rate > 60 || output.bit_rate < 1'000 || output.bit_rate > 12'000'000) { + if (!piik::capture::ValidVideoFormat(output.width, output.height, output.frame_rate) || + output.bit_rate < 1'000 || output.bit_rate > 12'000'000) { Fail("argument-output", "output profile exceeds codec bounds"); } if (arguments.mode == ProductArguments::Mode::video && @@ -1674,7 +1674,8 @@ ComPtr OwnDecodedTexture(const DeviceContext& device, IMFSample D3D11_TEXTURE2D_DESC description{}; original->GetDesc(&description); if (description.Format != DXGI_FORMAT_NV12 || description.Width < width || description.Height < height || - description.Width > 2560 || description.Height > 1440) Fail("decoded-texture-size", "decoder texture exceeds source bounds"); + !piik::capture::ValidCodedVideoSize(description.Width, description.Height)) + Fail("decoded-texture-size", "decoder texture exceeds source bounds"); description.ArraySize = description.MipLevels = 1; description.Usage = D3D11_USAGE_DEFAULT; description.CPUAccessFlags = description.MiscFlags = 0; diff --git a/native/capture/windows/output_worker.test.cpp b/native/capture/windows/output_worker.test.cpp index f74a6cc4d..bd5aeb170 100644 --- a/native/capture/windows/output_worker.test.cpp +++ b/native/capture/windows/output_worker.test.cpp @@ -315,11 +315,13 @@ int main() { assert(profile.h264_level() == 51 && profile.profile_level_id() == "42c033"); profile = VideoProfile{1920, 1080, 60, 8'000'000}; assert(profile.h264_level() == 42); - for (const SIZE size : {SIZE{1920, 1200}, SIZE{720, 1280}, SIZE{4, 4}}) { + for (const SIZE size : {SIZE{1920, 1200}, SIZE{720, 1280}, SIZE{4, 4}, + SIZE{3840, 2160}, SIZE{2160, 3840}, SIZE{3000, 2764}}) { ValidateVideoProfile(VideoProfile{static_cast(size.cx), static_cast(size.cy), 30, 3'000'000}); } for (const SIZE size : {SIZE{0, 720}, SIZE{2, 720}, SIZE{1280, 2}, SIZE{1279, 720}, - SIZE{1280, 719}, SIZE{2562, 1440}, SIZE{2560, 1442}, SIZE{3840, 2160}}) { + SIZE{1280, 719}, SIZE{3842, 2160}, SIZE{2160, 3842}, + SIZE{3840, 2162}, SIZE{3000, 2766}}) { try { ValidateVideoProfile(VideoProfile{static_cast(size.cx), static_cast(size.cy), 30, 3'000'000}); assert(false); @@ -327,6 +329,14 @@ int main() { assert(error.stage() == "argument-profile"); } } + assert(piik::capture::ValidVideoFormat(3840, 2160, 30)); + assert(!piik::capture::ValidVideoFormat(3840, 2160, 31)); + assert(!piik::capture::ValidVideoFormat(2160, 3840, 60)); + assert(!piik::capture::ValidVideoFormat(UINT32_MAX, UINT32_MAX, UINT32_MAX)); + assert(piik::capture::ValidCodedVideoSize(3840, 2176)); + assert(piik::capture::ValidCodedVideoSize(2176, 3840)); + assert(!piik::capture::ValidCodedVideoSize(3840, 3840)); + assert(!piik::capture::ValidCodedVideoSize(UINT32_MAX, UINT32_MAX)); CheckEncoderCandidates(); CheckH264ProfileCompatibility(); CheckPrimaryFailureDetails(); diff --git a/native/capture/windows/video_bounds.h b/native/capture/windows/video_bounds.h new file mode 100644 index 000000000..79fac850a --- /dev/null +++ b/native/capture/windows/video_bounds.h @@ -0,0 +1,30 @@ +#pragma once + +#include + +namespace piik::capture { + +inline constexpr uint32_t kMaxVideoDimension = 3840; +inline constexpr uint64_t kMaxVideoPixels = 3840 * 2160; +// H.264 Level 5.1 coded storage, including padding outside the visible crop. +inline constexpr uint32_t kMaxCodedVideoDimension = 4096; +inline constexpr uint64_t kMaxCodedVideoPixels = 36'864 * 16 * 16; + +inline bool ValidVideoSize(uint32_t width, uint32_t height) { + return width >= 2 && width <= kMaxVideoDimension && !(width & 1) && + height >= 2 && height <= kMaxVideoDimension && !(height & 1) && + static_cast(width) * height <= kMaxVideoPixels; +} + +inline bool ValidVideoFormat(uint32_t width, uint32_t height, uint32_t fps) { + return ValidVideoSize(width, height) && fps >= 1 && fps <= 60 && + static_cast((width + 15) / 16) * ((height + 15) / 16) * fps <= 983'040; +} + +inline bool ValidCodedVideoSize(uint32_t width, uint32_t height) { + return width >= 2 && width <= kMaxCodedVideoDimension && !(width & 1) && + height >= 2 && height <= kMaxCodedVideoDimension && !(height & 1) && + static_cast(width) * height <= kMaxCodedVideoPixels; +} + +} // namespace piik::capture diff --git a/native/capture/windows/vp8_decoder.h b/native/capture/windows/vp8_decoder.h index cb57c4eb4..d7246aac8 100644 --- a/native/capture/windows/vp8_decoder.h +++ b/native/capture/windows/vp8_decoder.h @@ -1,5 +1,7 @@ #pragma once +#include "video_bounds.h" + #include #include @@ -37,7 +39,7 @@ class Vp8Decoder final { vpx_codec_stream_info_t info{}; info.sz = sizeof(info); if (vpx_codec_peek_stream_info(vpx_codec_vp8_dx(), data.data(), static_cast(data.size()), &info) != VPX_CODEC_OK || - info.w < 2 || info.h < 2 || info.w > 2560 || info.h > 1440 || (info.w & 1) || (info.h & 1)) { + !ValidVideoSize(info.w, info.h)) { throw std::runtime_error("VP8 decoder input dimensions exceed the source bound"); } } @@ -48,7 +50,7 @@ class Vp8Decoder final { const auto* image = vpx_codec_get_frame(&codec_, &iterator); if (!image) return {}; const uint32_t width = image->d_w, height = image->d_h; - if (width < 2 || height < 2 || width > 2560 || height > 1440 || (width & 1) || (height & 1) || + if (!ValidVideoSize(width, height) || image->fmt != VPX_IMG_FMT_I420 || image->stride[0] < static_cast(width) || image->stride[1] < static_cast(width / 2) || image->stride[2] < static_cast(width / 2)) { throw std::runtime_error("VP8 decoder output format is unsupported"); From 736f9324d9ee9fab20672efa800c7665cf5b34e8 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:56:08 +0800 Subject: [PATCH 074/262] feat(media): bound 4K controls by capture capabilities --- docs/research/native-client-media.md | 16 +++++--- docs/standards/media-quality.md | 15 +++++--- docs/status.md | 12 +++--- docs/todo.md | 18 +++------ internal/app/app.go | 22 ++++++----- internal/app/loopback/server.go | 24 ++++++------ internal/app/nativecapture/protocol.go | 15 ++++---- scripts/app-native-host-gate.ts | 27 +++++++++++++- .../components/living/QualityPresets.tsx | 23 ++++++++---- src/client/locales/en.ts | 2 +- src/client/locales/zh.ts | 2 +- src/client/media/quality.ts | 37 ++++++++++++++++++- src/client/native/wire.ts | 2 + src/client/pages/ControlsPreview.tsx | 4 +- src/client/pages/HostPage.tsx | 22 +++++------ tests/native-client.test.ts | 8 ++++ tests/quality.test.ts | 21 +++++++++++ 17 files changed, 185 insertions(+), 85 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index e2d428e9e..47d27e702 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -571,12 +571,15 @@ presets. A source ceiling must also yield a valid half-size output. Bitrate and output-count bounds remain unchanged. Candidate signaling and Native control now carry numeric width/height ceilings, with custom input beside the existing UI shortcuts. Descriptive wire bounds are independent of the current implementation maximum; -App discovery advertises per-axis maxima, and Native rejects unsupported +App discovery advertises axis, pixel-count and macroblock-rate limits, and Native rejects unsupported size/rate combinations as operation failures without retiring the control session. Removing an enum is not permission to allocate arbitrary media resources. -The Windows H.264 UI gate applies a non-preset 960x600 ceiling during a live -share and observes fitted output without replacing the Viewer media object. +The Windows H.264 UI gate switches from 1080p60 to the 4K ceiling with the +30 fps limit, then applies a non-preset 960x600 ceiling during the live share. +It observes fitted output without replacing the Viewer media object. The +captured window is smaller than 4K, so this verifies settings/lifecycle, not +physical 4K capture. Preset changes, paused updates, source recovery/replacement and subsequent sharing still pass. Value-based comparison prevents an identical deserialized size from turning an audio-only edit into a video update. Numeric preference @@ -603,9 +606,10 @@ VP8 derivation produced the lower output from the 4K input. The Browser H.264 SF a synthetic 3840x2160/30 source passed first-picture delivery, an explicit lower profile, audio-only edits with retained connections, subscription/page recovery and retirement. Windows native checks and Linux headless build/checks pass; -macOS SDK and physical 4K capture are not verified here. The UI ceiling remains -2560x1440 until combined capability controls and whole-route acceptance are -complete. These checks do not establish 4K60, sustained device performance or +macOS SDK and physical 4K capture are not verified here. Controls now use the +combined limits, reject oversized pixel totals and cap frame rate on size changes. +Whole-route and cross-device acceptance remain open. These checks do not +establish 4K60, sustained device performance or universal portrait decoding support. The proposed boundary follows the diff --git a/docs/standards/media-quality.md b/docs/standards/media-quality.md index d5857cdeb..2c2ba8349 100644 --- a/docs/standards/media-quality.md +++ b/docs/standards/media-quality.md @@ -127,12 +127,14 @@ The three recommended profiles are ceilings, not delivery guarantees: | `1080p60` | 1920x1080 at 60 fps | 8 Mbps | `1080p30` is the default. The 2.0 candidate carries numeric width/height ceilings; -480p, 720p, 1080p and 1440p remain UI shortcuts. Advanced picture settings also -accept custom even dimensions within the current 2560x1440 UI bound. -Width and height form one applied edit. Larger wire-valid requests do not +480p, 720p, 1080p, 1440p and 4K are UI shortcuts. Advanced picture settings also +accept custom even dimensions within the combined implementation bounds below. +Width and height form one applied edit. A size change caps the requested frame +rate in the same quality operation when the larger size requires a lower limit. +Larger wire-valid requests do not authorize an adapter to exceed its local capture, codec or allocation limits. Frame rate, bitrate, and `maintain-resolution | balanced | -maintain-framerate` remain independent controls. Display video uses the standard +maintain-framerate` retain their controls. Display video uses the standard `contentHint = "motion"` for game motion. Native H264 retains Constrained Baseline. Capture validates the coding-tool subset under RFC 6184, including equivalent Constrained Baseline constraint bytes, and reports @@ -151,7 +153,10 @@ Level 5.1: 983,040 16x16 macroblocks per second, with at most 60 fps. This admit landscape/portrait 4K30 and existing 1440p60, not 4K60. Decoder alignment has a separate coded-storage bound; reference-picture storage is bounded before Native derivation. These are admission limits, not a device performance promise. -The UI remains within its previous ceiling pending whole-route acceptance. +App discovery reports axis, pixel-count and output-rate limits through the +existing native health description. Controls use the intersection with local +implementation limits; a remote claim cannot raise those limits. Whole-route +and physical-device acceptance remain separate from available controls. Framework-driven downscaling under bandwidth or device pressure is valid within the selected degradation preference. A low decoded resolution alone is not a diff --git a/docs/status.md b/docs/status.md index 8fcc633a7..015bb9549 100644 --- a/docs/status.md +++ b/docs/status.md @@ -66,13 +66,11 @@ Browser/server v24 and Native control v10 replace the preset fields; capture v7 and SQLite schema 2 remain unchanged. Matched candidate App/Web/Server builds are required; 1.x participants are rejected by the existing compatibility checks. -Custom width/height controls retain the current implementation ceiling. -Windows and Linux output now preserve source shape within that ceiling. -Native admission now has axis-independent size, pixel-count and output-rate -bounds for 4K30; local Windows encoding, decoding and derived-output checks pass. -Browser H.264 4K30 also passes the existing SFU flow with a synthetic source. -Browser/macOS geometry, 4K controls and whole-route/device acceptance, and -additional codecs remain in the media assessment. +Custom width/height controls apply combined size and frame-rate limits, admitting +4K30. Windows and Linux output preserve source shape within the selected ceiling. +The [media assessment](./research/native-client-media.md#media-capability-extension-assessment) +owns local codec, derived-output, SFU and live-control evidence. Browser/macOS +geometry, whole-route/device acceptance and additional codecs remain open. Native Magicsock remains an isolated experiment with incomplete product integration and unresolved recovery cases. These proposals do not yet change the accepted contract or transport. A small external test is proposed before diff --git a/docs/todo.md b/docs/todo.md index ba3d02fc0..489d0af47 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -47,19 +47,11 @@ Implementation needs a concrete useful result within the product model. - [ ] **Media parameters and capabilities for 2.0.** Follow the [media assessment](./research/native-client-media.md#media-capability-extension-assessment): separate UI presets, numeric requests, bounded implementation capabilities and - actual output. Numeric audio admission, storage and the compact custom input - implement [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s use case, with - local Browser/direct/relay and Browser/native SFU evidence. Numeric video - requests and custom width/height controls now pass the existing Windows live - capture gate within current implementation limits. Next assess - source-aspect-preserving geometry and 4K through the same owners. Windows and - Linux outputs now fit the source without a fixed padded landscape canvas; - Native axis-independent 4K30 admission and bounded Windows codec/derived-output - checks now pass, as does Browser H.264 4K30 over the existing SFU gate with a - synthetic source. Next align controls with combined size/rate limits and verify - Native/Browser mixed routes, source/live changes and limited receivers. - Browser/macOS geometry and physical 4K capture remain open. Preserve - safety limits and framework-owned adaptation. The + actual output. Numeric audio and video controls are implemented; the assessment + owns local capture, codec, derived-output and SFU evidence. Next verify + Native/Browser mixed routes, 4K source/live changes and limited receivers. + Browser/macOS geometry and physical 4K capture remain open. Preserve safety + limits and framework-owned adaptation. The [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment) retain explicit receive-limit enforcement and the current 320 kbps sender ceiling; physical listening and impaired-network evidence remain open. diff --git a/internal/app/app.go b/internal/app/app.go index b19955222..6966ecca4 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -631,16 +631,18 @@ func discoverNativeMedia(ctx context.Context, configuredPath string) nativeRunti MaxVideoResolution: protocol.QualityResolution{ Width: nativecapture.MaxVideoDimension, Height: nativecapture.MaxVideoDimension, }, - MaxAudioBitrate: nativeaudio.MaxSendBitrate, - ReceiverReuse: true, - Video: summary.Video, - CaptureBorderControl: summary.CaptureBorderControl, - Microphone: summary.Microphone, - ProcessAudio: summary.ProcessAudio, - ProcessAudioExclusion: summary.ProcessAudioExclusion, - SystemAudio: summary.SystemAudio, - HardwareH264: summary.HardwareH264, - SoftwareVP8: summary.SoftwareVP8, + MaxVideoPixels: nativecapture.MaxVideoPixels, + MaxVideoMacroblocksPerSecond: nativecapture.MaxVideoMacroblocksPerSecond, + MaxAudioBitrate: nativeaudio.MaxSendBitrate, + ReceiverReuse: true, + Video: summary.Video, + CaptureBorderControl: summary.CaptureBorderControl, + Microphone: summary.Microphone, + ProcessAudio: summary.ProcessAudio, + ProcessAudioExclusion: summary.ProcessAudioExclusion, + SystemAudio: summary.SystemAudio, + HardwareH264: summary.HardwareH264, + SoftwareVP8: summary.SoftwareVP8, }, } } diff --git a/internal/app/loopback/server.go b/internal/app/loopback/server.go index e7617ac41..7fa232116 100644 --- a/internal/app/loopback/server.go +++ b/internal/app/loopback/server.go @@ -73,17 +73,19 @@ type Health struct { } type NativeMediaCapabilities struct { - MaxVideoResolution protocol.QualityResolution `json:"maxVideoResolution,omitzero"` - MaxAudioBitrate int `json:"maxAudioBitrate,omitempty"` - ReceiverReuse bool `json:"receiverReuse,omitempty"` - Video bool `json:"video"` - CaptureBorderControl bool `json:"captureBorderControl,omitempty"` - Microphone bool `json:"microphone,omitempty"` - ProcessAudio bool `json:"processAudio"` - ProcessAudioExclusion bool `json:"processAudioExclusion,omitempty"` - SystemAudio bool `json:"systemAudio"` - HardwareH264 bool `json:"hardwareH264"` - SoftwareVP8 bool `json:"softwareVP8"` + MaxVideoResolution protocol.QualityResolution `json:"maxVideoResolution,omitzero"` + MaxVideoPixels int `json:"maxVideoPixels,omitempty"` + MaxVideoMacroblocksPerSecond int `json:"maxVideoMacroblocksPerSecond,omitempty"` + MaxAudioBitrate int `json:"maxAudioBitrate,omitempty"` + ReceiverReuse bool `json:"receiverReuse,omitempty"` + Video bool `json:"video"` + CaptureBorderControl bool `json:"captureBorderControl,omitempty"` + Microphone bool `json:"microphone,omitempty"` + ProcessAudio bool `json:"processAudio"` + ProcessAudioExclusion bool `json:"processAudioExclusion,omitempty"` + SystemAudio bool `json:"systemAudio"` + HardwareH264 bool `json:"hardwareH264"` + SoftwareVP8 bool `json:"softwareVP8"` } type Server struct { diff --git a/internal/app/nativecapture/protocol.go b/internal/app/nativecapture/protocol.go index 236e38eff..e38dc0c53 100644 --- a/internal/app/nativecapture/protocol.go +++ b/internal/app/nativecapture/protocol.go @@ -11,12 +11,13 @@ import ( ) const ( - envelopeHeaderBytes = 32 - maxMediaBytes = encoded.MaxAccessUnitBytes - maxStatusBytes = 4 * 1024 - maxControlBytes = 64 - MaxVideoDimension = 3840 - MaxVideoPixels = 3840 * 2160 + envelopeHeaderBytes = 32 + maxMediaBytes = encoded.MaxAccessUnitBytes + maxStatusBytes = 4 * 1024 + maxControlBytes = 64 + MaxVideoDimension = 3840 + MaxVideoPixels = 3840 * 2160 + MaxVideoMacroblocksPerSecond = 983_040 // Coded pictures may include alignment outside the visible crop. Keep this // separate from visible dimensions and within H.264 Level 5.1 storage. MaxCodedVideoDimension = 4096 @@ -34,7 +35,7 @@ func ValidVideoSize(width, height uint32) bool { // ceiling (H.264 Level 5.1), including non-preset shapes and frame rates. func ValidVideoFormat(width, height, framerate uint32) bool { return ValidVideoSize(width, height) && framerate >= 1 && framerate <= 60 && - uint64((width+15)/16)*uint64((height+15)/16)*uint64(framerate) <= 983_040 + uint64((width+15)/16)*uint64((height+15)/16)*uint64(framerate) <= MaxVideoMacroblocksPerSecond } func ValidCodedVideoSize(width, height uint64) bool { diff --git a/scripts/app-native-host-gate.ts b/scripts/app-native-host-gate.ts index 5c80f04a5..0bf8c6053 100644 --- a/scripts/app-native-host-gate.ts +++ b/scripts/app-native-host-gate.ts @@ -82,6 +82,7 @@ interface GateResult { qualityControlsEnabled: boolean; livePresetChanges: number; liveQualityChanged: boolean; + fourKQualityChanged: boolean; customQualityChanged: boolean; pausedQualityChanged: boolean; backgroundProfileRecovery: boolean | null; @@ -633,6 +634,7 @@ async function main(): Promise { qualityControlsEnabled: false, livePresetChanges: 0, liveQualityChanged: false, + fourKQualityChanged: false, customQualityChanged: false, pausedQualityChanged: false, backgroundProfileRecovery: mode === "local" && sourceKind === "window" ? false : null, @@ -1020,6 +1022,29 @@ async function main(): Promise { result.viewerWidth = viewerState.width; result.viewerHeight = viewerState.height; await assertVideoCodec(cdp, viewer, actualCodec); + stage = "native-4k-quality"; + await clickHostControl(cdp, host, `document.querySelectorAll('button.lr-tile')[2]`); + await waitForValue(deadline => evaluate(cdp!, host, + `document.querySelector('.lr-slider input[type="range"]')?.value === '60' && + document.querySelector('.lr-sharing-panel')?.getAttribute('aria-busy') === 'false'`, + deadline), Boolean, 20_000); + const framesBefore4K = await evaluate(cdp, viewer, + `document.querySelector('video')?.getVideoPlaybackQuality().totalVideoFrames ?? 0`, Date.now() + 5_000); + await evaluate(cdp, host, + `document.querySelector('button.lr-chip[aria-label="4K"]')?.click()`, Date.now() + 5_000); + await waitForValue(deadline => evaluate(cdp!, host, `(() => { + const fps = document.querySelector('.lr-slider input[type="range"]'); + const width = document.querySelector('.lr-video-size input[name="width"]'); + const height = document.querySelector('.lr-video-size input[name="height"]'); + return fps?.max === '30' && fps.value === '30' && width?.value === '3840' && height?.value === '2160'; + })()`, deadline), Boolean, 10_000); + result.fourKQualityChanged = await waitForValue(deadline => evaluate(cdp!, viewer, `(() => { + const video = document.querySelector('video'); + return Boolean(video && window.__piikGateFitsProfile(3840, 2160) && + video.srcObject === window.__piikGateMedia && + video.getVideoPlaybackQuality().totalVideoFrames >= ${framesBefore4K + 10}); + })()`, deadline), Boolean, 20_000); + await assertVideoCodec(cdp, viewer, actualCodec); stage = "native-custom-quality"; const framesBeforeCustom = await evaluate(cdp, viewer, `document.querySelector('video')?.getVideoPlaybackQuality().totalVideoFrames ?? 0`, Date.now() + 5_000); @@ -1493,7 +1518,7 @@ async function main(): Promise { result.viewerFrames >= 30 && result.viewerWidth > 0 && result.viewerWidth <= 2560 && result.viewerHeight > 0 && result.viewerHeight <= 1440 && result.codecPreserved : result.viewerConnected && result.viewerFrames >= 30 && - result.qualityControlsEnabled && result.liveQualityChanged && result.customQualityChanged && + result.qualityControlsEnabled && result.liveQualityChanged && result.fourKQualityChanged && result.customQualityChanged && result.backgroundProfileRecovery !== false && result.sourceWindowRecovery !== false && result.livePresetChanges === 2 && diff --git a/src/client/components/living/QualityPresets.tsx b/src/client/components/living/QualityPresets.tsx index fe56be928..7765e3e59 100644 --- a/src/client/components/living/QualityPresets.tsx +++ b/src/client/components/living/QualityPresets.tsx @@ -1,15 +1,16 @@ import { Fragment } from "react"; import { QUALITY_PROFILES, QUALITY_PROFILE_KEYS, QUALITY_RESOLUTIONS, qualityResolutionEqual, - type QualityProfileId, type QualityResolution, + captureFramerateCeiling, VIDEO_CAPTURE_LIMITS, type VideoCaptureLimits, type QualityProfileId, type QualityResolution, } from "../../media/quality"; import { useCopy } from "../../ui/copy"; import { Chip } from "./primitives"; import { QualityTileGlyph } from "./QualityTileGlyph"; import { Tooltip } from "./Tooltip"; -export function QualityPresets({ selected, disabled = false, busy = false, onSelect }: { +export function QualityPresets({ selected, limits = VIDEO_CAPTURE_LIMITS, disabled = false, busy = false, onSelect }: { selected: QualityProfileId | null; + limits?: VideoCaptureLimits; disabled?: boolean; busy?: boolean; onSelect: (id: QualityProfileId) => void; @@ -22,7 +23,8 @@ export function QualityPresets({ selected, disabled = false, busy = false, onSel return @@ -31,9 +33,9 @@ export function QualityPresets({ selected, disabled = false, busy = false, onSel
; } -export function ResolutionControl({ value, maximum, disabled, onChange }: { +export function ResolutionControl({ value, limits, disabled, onChange }: { value: QualityResolution; - maximum: QualityResolution; + limits: VideoCaptureLimits; disabled: boolean; onChange: (resolution: QualityResolution) => void; }) { @@ -50,6 +52,10 @@ export function ResolutionControl({ value, maximum, disabled, onChange }: { return; } const next = { width: width.valueAsNumber, height: height.valueAsNumber }; + if (captureFramerateCeiling(next, limits) < 15) { + reset(container); + return; + } if (!qualityResolutionEqual(value, next)) onChange(next); }; return
{Object.entries(QUALITY_RESOLUTIONS).map(([label, resolution]) => maximum.width || resolution.height > maximum.height} + disabled={disabled || captureFramerateCeiling(resolution, limits) < 15} title={label} hint="hint-quality" onClick={() => onChange(resolution)}>{label})}
{axes.map((axis, index) => {index === 1 ? : null} )} diff --git a/src/client/locales/en.ts b/src/client/locales/en.ts index 79a16c143..95f3a59fd 100644 --- a/src/client/locales/en.ts +++ b/src/client/locales/en.ts @@ -361,7 +361,7 @@ export const en: Record = { "host.advanced.audioBitrate": "Audio bitrate limit", "host.video.width": "Width limit in pixels", "host.video.height": "Height limit in pixels", - "host.video.resolutionHint": "Width × height limits: {width} × {height} pixels. Source proportions are preserved.", + "host.video.resolutionHint": "Width ≤ {width} px; height ≤ {height} px; total ≤ {pixels} pixels. Source proportions are preserved; the FPS limit follows the size.", "host.audio.bitrateHint": "Audio bitrate limit: {min}–{max} kbps.", "host.advanced.audio.saver": "Standard", "host.advanced.audio.music": "Music", diff --git a/src/client/locales/zh.ts b/src/client/locales/zh.ts index 62480a4fa..f17f1bbc4 100644 --- a/src/client/locales/zh.ts +++ b/src/client/locales/zh.ts @@ -361,7 +361,7 @@ export const zh = { "host.advanced.audioBitrate": "音频码率上限", "host.video.width": "宽度上限(像素)", "host.video.height": "高度上限(像素)", - "host.video.resolutionHint": "宽 × 高上限:{width} × {height} 像素。保留来源比例。", + "host.video.resolutionHint": "宽 ≤ {width},高 ≤ {height} 像素;总像素 ≤ {pixels}。保留来源比例,帧率上限随尺寸调整。", "host.audio.bitrateHint": "音频码率上限:{min}–{max} kbps。", "host.advanced.audio.saver": "普通", "host.advanced.audio.music": "音乐", diff --git a/src/client/media/quality.ts b/src/client/media/quality.ts index 4a93b4c58..6f521c751 100644 --- a/src/client/media/quality.ts +++ b/src/client/media/quality.ts @@ -21,8 +21,40 @@ export type { export type QualityProfile = QualitySettings; export const STARTUP_VIDEO_ENCODED_FRAMES = 5; -// Current product output boundary, independent of wire admission and UI shortcuts. -export const MAX_CAPTURE_RESOLUTION = { width: 2560, height: 1440 } as const; +export interface VideoCaptureLimits { + maxVideoResolution: QualityResolution; + maxVideoPixels: number; + maxVideoMacroblocksPerSecond: number; +} + +// Current output boundary, independent of descriptive wire bounds and UI presets. +// The common H.264 Level 5.1 path admits 4K30 and 1440p60 in either orientation. +export const VIDEO_CAPTURE_LIMITS: VideoCaptureLimits = { + maxVideoResolution: { width: 3840, height: 3840 }, + maxVideoPixels: 3840 * 2160, + maxVideoMacroblocksPerSecond: 983_040, +}; + +export function videoCaptureLimits(advertised?: Partial): VideoCaptureLimits { + const local = VIDEO_CAPTURE_LIMITS; + return { + maxVideoResolution: { + width: Math.min(local.maxVideoResolution.width, advertised?.maxVideoResolution?.width ?? local.maxVideoResolution.width), + height: Math.min(local.maxVideoResolution.height, advertised?.maxVideoResolution?.height ?? local.maxVideoResolution.height), + }, + maxVideoPixels: Math.min(local.maxVideoPixels, advertised?.maxVideoPixels ?? local.maxVideoPixels), + maxVideoMacroblocksPerSecond: Math.min(local.maxVideoMacroblocksPerSecond, + advertised?.maxVideoMacroblocksPerSecond ?? local.maxVideoMacroblocksPerSecond), + }; +} + +export function captureFramerateCeiling(size: QualityResolution, limits = VIDEO_CAPTURE_LIMITS): number { + if (![size.width, size.height].every(value => Number.isInteger(value) && value >= 4 && value % 2 === 0) + || size.width > limits.maxVideoResolution.width || size.height > limits.maxVideoResolution.height + || size.width * size.height > limits.maxVideoPixels) return 0; + return Math.min(60, Math.floor(limits.maxVideoMacroblocksPerSecond / + (Math.ceil(size.width / 16) * Math.ceil(size.height / 16)))); +} export const QUALITY_PROFILES = { "720p30": { @@ -54,6 +86,7 @@ export const QUALITY_RESOLUTIONS = { "720p": { width: 1280, height: 720 }, "1080p": { width: 1920, height: 1080 }, "1440p": { width: 2560, height: 1440 }, + "4K": { width: 3840, height: 2160 }, } as const satisfies Record; export function qualityResolutionEqual(left: QualityResolution, right: QualityResolution): boolean { diff --git a/src/client/native/wire.ts b/src/client/native/wire.ts index a83e95652..b91cb0c5d 100644 --- a/src/client/native/wire.ts +++ b/src/client/native/wire.ts @@ -35,6 +35,8 @@ export const nativeHealthSchema = nativeDiscoveryIdentitySchema.extend({ .object({ maxAudioBitrate: maxAudioBitrateSchema.optional(), maxVideoResolution: qualityResolutionSchema.optional(), + maxVideoPixels: z.number().int().min(16).max(8192 * 8192).optional(), + maxVideoMacroblocksPerSecond: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).optional(), receiverReuse: z.boolean().default(false), video: z.boolean().default(false), processAudio: z.boolean().default(false), diff --git a/src/client/pages/ControlsPreview.tsx b/src/client/pages/ControlsPreview.tsx index 8f2aca9e4..c639369b2 100644 --- a/src/client/pages/ControlsPreview.tsx +++ b/src/client/pages/ControlsPreview.tsx @@ -21,7 +21,7 @@ import { useTheaterMode } from "../components/living/use-theater-mode"; import { QualityPresets, ResolutionControl } from "../components/living/QualityPresets"; import { RoomCodeInput } from "../components/living/RoomCodeInput"; -import { MAX_CAPTURE_RESOLUTION, QUALITY_PROFILES, type QualityProfileId, type QualityResolution } from "../media/quality"; +import { VIDEO_CAPTURE_LIMITS, QUALITY_PROFILES, type QualityProfileId, type QualityResolution } from "../media/quality"; import type { loadCameraPreviews } from "../media/camera-previews"; import { Tooltip } from "../components/living/Tooltip"; import { StatusIndicator } from "../components/living/StatusIndicator"; @@ -119,7 +119,7 @@ export function ControlsPreview() {
{ setPreset(id); setResolution(QUALITY_PROFILES[id].resolution); }} />} - picture={ { setResolution(value); setPreset(null); }} />} audio={<>
@@ -3835,7 +3833,7 @@ export function HostPage({
void changeQuality({ ...QUALITY_PROFILES[id], maxAudioBitrate: screenAudioBitrate(advancedQualityRef.current.maxAudioBitrate), @@ -3849,9 +3847,11 @@ export function HostPage({ - changeAdvancedQuality({ resolution })} /> + onChange={resolution => changeAdvancedQuality({ resolution, + maxFramerate: Math.min(advancedQualityRef.current.maxFramerate, captureFramerateCeiling(resolution, videoLimits)), + })} />
@@ -3866,8 +3866,8 @@ export function HostPage({ { nativeMedia: { video: true, hardwareH264: true }, }; + it("reads numeric video capability descriptions and rejects malformed limits", () => { + const nativeMedia = { ...health.nativeMedia, maxVideoResolution: { width: 3840, height: 3840 }, + maxVideoPixels: 3840 * 2160, maxVideoMacroblocksPerSecond: 983_040 }; + expect(nativeHealthSchema.parse({ ...health, nativeMedia }).nativeMedia).toMatchObject(nativeMedia); + expect(nativeHealthSchema.safeParse({ ...health, nativeMedia: { ...nativeMedia, maxVideoPixels: 0 } }).success).toBe(false); + expect(nativeHealthSchema.safeParse({ ...health, nativeMedia: { ...nativeMedia, maxVideoMacroblocksPerSecond: -1 } }).success).toBe(false); + }); + it.each(["prompt", "denied"] as const)("does not hold Viewer reception for %s permission and observes a later grant", async (state) => { vi.stubGlobal("window", { setTimeout, clearTimeout, location: { hostname: "piik.example" } }); const query = vi.fn().mockResolvedValue({ state }); diff --git a/tests/quality.test.ts b/tests/quality.test.ts index 7de190a85..a65ab7b37 100644 --- a/tests/quality.test.ts +++ b/tests/quality.test.ts @@ -5,6 +5,8 @@ import { applyCaptureProfile, applyVideoCaptureProfile, captureDisplay, + captureFramerateCeiling, + videoCaptureLimits, configureScreenAudioSender, configureVideoSender, matchingQualityProfileId, @@ -50,6 +52,25 @@ afterEach(() => { }); describe("realtime quality controls", () => { + it("bounds custom size, orientation and frame rate together before capture", () => { + expect(captureFramerateCeiling(QUALITY_RESOLUTIONS["4K"])).toBe(30); + expect(captureFramerateCeiling({ width: 2160, height: 3840 })).toBe(30); + expect(captureFramerateCeiling(QUALITY_RESOLUTIONS["1440p"])).toBe(60); + expect(captureFramerateCeiling({ width: 1440, height: 2560 })).toBe(60); + expect(captureFramerateCeiling({ width: 3840, height: 2100 })).toBe(31); + for (const size of [{ width: 3840, height: 3840 }, { width: 1921, height: 1080 }, + { width: 4.5, height: 4 }, { width: Infinity, height: 4 }, { width: 2, height: 4 }]) { + expect(captureFramerateCeiling(size)).toBe(0); + } + const limited = videoCaptureLimits({ maxVideoResolution: { width: 1920, height: 1080 }, + maxVideoPixels: 1920 * 1080, maxVideoMacroblocksPerSecond: 245_760 }); + expect(captureFramerateCeiling(QUALITY_RESOLUTIONS["4K"], limited)).toBe(0); + expect(captureFramerateCeiling(QUALITY_RESOLUTIONS["1080p"], limited)).toBe(30); + expect(captureFramerateCeiling(QUALITY_RESOLUTIONS["720p"], limited)).toBe(60); + expect(videoCaptureLimits({ maxVideoResolution: { width: 8192, height: 8192 }, + maxVideoPixels: 8192 * 8192, maxVideoMacroblocksPerSecond: 9_000_000 })).toEqual(videoCaptureLimits()); + }); + it("keeps exactly three recommended profiles with 1080p30 as the default", () => { expect(Object.keys(QUALITY_PROFILES)).toEqual([ "720p30", From fa61093ab41712fbf75fc48c62c0d35ca3bca637 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:24:11 +0800 Subject: [PATCH 075/262] test(media): cover numeric source geometry and mixed fanout --- docs/research/browser-local-encoding-pool.md | 7 ++++++ docs/research/native-client-media.md | 15 +++++++++--- docs/status.md | 5 ++-- docs/todo.md | 6 +++-- scripts/app-browser-fanout-gate.ts | 25 +++++++++++++++----- scripts/browser-local-pool-page.js | 25 ++++++++++++-------- scripts/browser-local-pool-probe.ts | 12 +++++++--- scripts/media-gate-geometry.ts | 11 +++++++++ tests/browser-gate-harness.test.ts | 10 ++++++++ 9 files changed, 90 insertions(+), 26 deletions(-) diff --git a/docs/research/browser-local-encoding-pool.md b/docs/research/browser-local-encoding-pool.md index a462562cd..50e3d1eff 100644 --- a/docs/research/browser-local-encoding-pool.md +++ b/docs/research/browser-local-encoding-pool.md @@ -677,9 +677,16 @@ npx tsx scripts/browser-local-pool-probe.ts carrier --1080 --background npx tsx scripts/browser-local-pool-probe.ts carrier --display --quiet-start --1080 --h264 --av --lifecycle npx tsx scripts/browser-local-pool-probe.ts ordinary --display --quiet-start --1080 --av --lifecycle npx tsx scripts/browser-local-pool-probe.ts carrier --display --1080 --h264 --av --network --auto --rate=400000 --preference=balanced +npx tsx scripts/browser-local-pool-probe.ts carrier --display --size=3840x2160 --h264 --single --lifecycle +npx tsx scripts/browser-local-pool-probe.ts carrier --size=2160x3840 --h264 --relay --late ``` Add `--h264`, `--single` or `--late` for the corresponding case. Set +`--size=WIDTHxHEIGHT` for a numeric source/quality ceiling within current 30 fps +capture bounds; this also requests that window size for real tab capture. +Chrome's content area and capture toolbar can make the actual surface smaller. +The pool check expects one output owner per compatible allocation, not one +owner for children with different bandwidth budgets. Set `--network --auto --short-pulse` for a 45-second warmup, one-second constraint and 40-second recovery instead of the ordinary 14-second constraint. Set `CHROME_PATH` for another installed Chromium binary. `--auto` extends the diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 47d27e702..e2b15c7ef 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -608,9 +608,18 @@ profile, audio-only edits with retained connections, subscription/page recovery and retirement. Windows native checks and Linux headless build/checks pass; macOS SDK and physical 4K capture are not verified here. Controls now use the combined limits, reject oversized pixel totals and cap frame rate on size changes. -Whole-route and cross-device acceptance remain open. These checks do not -establish 4K60, sustained device performance or -universal portrait decoding support. + +Chrome 154 real portrait/landscape tab capture retained the source aspect through +the pooled output. The landscape check also passed live/quiet/paused profile +changes, source replacement and ordinary recovery after producer failure. +A synthetic 2160x3840 Browser relay delivered fresh pictures to two children, +including a late connection. Separate 3840x2160 and 2160x3840 Browser-to-Native +fanout checks delivered about 30 fps to both Viewers and passed live reduction, +pause/resume, source-audio changes and App-exit recovery. Reproduce with +`PIIK_CLIENT_BROWSER_FANOUT_GATE=true`, `PIIK_CLIENT_GATE_VIDEO_SIZE=WIDTHxHEIGHT` +and `CHROME_PATH` set, then run `npm run gate:app-browser-fanout`. +These local checks do not establish 4K60, physical 4K/game +capture, sustained device performance, all Browser decoders or all route changes. The proposed boundary follows the [Media Capture specification](https://www.w3.org/TR/mediacapture-streams/#constrainable-interface): diff --git a/docs/status.md b/docs/status.md index 015bb9549..ff0bf8a74 100644 --- a/docs/status.md +++ b/docs/status.md @@ -69,8 +69,9 @@ remain unchanged. Matched candidate App/Web/Server builds are required; Custom width/height controls apply combined size and frame-rate limits, admitting 4K30. Windows and Linux output preserve source shape within the selected ceiling. The [media assessment](./research/native-client-media.md#media-capability-extension-assessment) -owns local codec, derived-output, SFU and live-control evidence. Browser/macOS -geometry, whole-route/device acceptance and additional codecs remain open. +owns local codec, derived-output, SFU, Browser/Native fanout and live-control +evidence. macOS geometry, cross-browser/device and route-change acceptance, and +additional codecs remain open. Native Magicsock remains an isolated experiment with incomplete product integration and unresolved recovery cases. These proposals do not yet change the accepted contract or transport. A small external test is proposed before diff --git a/docs/todo.md b/docs/todo.md index 489d0af47..26e0d1449 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -49,8 +49,10 @@ Implementation needs a concrete useful result within the product model. separate UI presets, numeric requests, bounded implementation capabilities and actual output. Numeric audio and video controls are implemented; the assessment owns local capture, codec, derived-output and SFU evidence. Next verify - Native/Browser mixed routes, 4K source/live changes and limited receivers. - Browser/macOS geometry and physical 4K capture remain open. Preserve safety + route changes, limited receivers and cross-browser/camera orientation behavior. + Browser display capture, Browser relay and Browser-to-Native fanout now have + bounded high-resolution evidence. macOS geometry and physical 4K/game capture + remain open. Preserve safety limits and framework-owned adaptation. The [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment) retain explicit receive-limit enforcement and the current 320 kbps sender diff --git a/scripts/app-browser-fanout-gate.ts b/scripts/app-browser-fanout-gate.ts index e534a31e6..531e881e3 100644 --- a/scripts/app-browser-fanout-gate.ts +++ b/scripts/app-browser-fanout-gate.ts @@ -8,6 +8,7 @@ import { waitForSample, waitForVersion, type PageHandle, } from "./browser-gate-harness"; import { readAppEndpoint } from "./app-gate-endpoint"; +import { fittedCaptureSize, matchesCaptureSize, parseVideoGateSize } from "./media-gate-geometry"; const ROOT = resolve(import.meta.dirname, ".."); const BUILD_ROOT = join(ROOT, "build/go-check"); @@ -17,6 +18,7 @@ const BUILD_ROOT = join(ROOT, "build/go-check"); const LAN_ADDRESS = "192.0.2.1"; const initialAudio = process.env.PIIK_CLIENT_GATE_AUDIO !== "false"; +const captureSize = parseVideoGateSize(process.env.PIIK_CLIENT_GATE_VIDEO_SIZE || "1920x1080"); const probe = String.raw`(() => { localStorage.setItem('piik:ui-lang', 'en'); localStorage.setItem('piik:ui-mode', 'text'); @@ -38,8 +40,8 @@ const probe = String.raw`(() => { let audio = ${initialAudio}, captures = 0; navigator.mediaDevices.getDisplayMedia = async () => { captures++; - const canvas = document.createElement('canvas'); canvas.width = 1920; canvas.height = 1080; - const context = canvas.getContext('2d'); let frame = 0; + const canvas = document.createElement('canvas'); canvas.width = ${captureSize.width}; canvas.height = ${captureSize.height}; + const context = canvas.getContext('2d'); context.scale(canvas.width/1920,canvas.height/1080); let frame = 0; const draw = () => { context.fillStyle = '#14242a'; context.fillRect(0,0,1920,1080); context.fillStyle = '#68dab3'; context.fillRect((frame++ * 17)%1800,100,120,400); context.fillStyle = '#ffffff'; context.font = '80px sans-serif'; context.fillText(String(frame),100,700); }; @@ -80,7 +82,7 @@ const probe = String.raw`(() => { } }); } const video = document.querySelector('video'); - return { outbound, encoded, received, captures, videoStats, feedback, width: video?.videoWidth || 0, + return { outbound, encoded, received, captures, videoStats, feedback, width: video?.videoWidth || 0, height: video?.videoHeight || 0, requests, errors, peers: pcs.map((p) => ({state:p.connectionState, tracks:p.getSenders().map((s)=>s.track?.kind)})), frames: video?.getVideoPlaybackQuality().totalVideoFrames || 0, sharing: Boolean(document.querySelector('button[aria-label="Stop sharing"]')), @@ -206,6 +208,12 @@ async function main(): Promise { await click(host, 'button[aria-controls="host-advanced-door"]'); stage = "share-codec"; await click(host, 'button[aria-label^="H264"]'); + await evaluate(cdp, host, `(() => { + const width = document.querySelector('.lr-video-size input[name="width"]'); + const height = document.querySelector('.lr-video-size input[name="height"]'); + width.focus(); width.value = '${captureSize.width}'; + height.focus(); height.value = '${captureSize.height}'; height.blur(); + })()`, Date.now()+5000); stage = "share-picker"; await click(host, 'button.lr-tv-big.is-action'); // This arm measures Native ingress. Browser capture can start without App @@ -240,8 +248,8 @@ async function main(): Promise { await waitForSample(() => read(viewer), (s) => s.width > 0 && s.frames >= 30, 30000); } stage = "shared-encode"; - await waitForSample(() => read(viewers[0]!), (s) => s.width===1920 && s.outbound===0, 30000); - await waitForSample(() => read(viewers[1]!), (s) => s.width===1920 && s.outbound===0, 30000); + for (const viewer of viewers) await waitForSample(() => read(viewer), + (s) => matchesCaptureSize(s.width, s.height, captureSize) && s.outbound===0, 30000); checks.host = await read(host); checks.viewers = await Promise.all(viewers.map(read)); if ((checks.host as any).outbound !== 1 || !(checks.viewers as any[]).every((v) => v.outbound === 0)) { @@ -255,7 +263,12 @@ async function main(): Promise { if (!(checks.viewerFps as number[]).every((fps) => fps>=27)) throw new Error("Viewer delivery did not sustain the 30 fps source"); stage = "quality"; await click(host, 'button[aria-label="720p"]'); - await waitForSample(() => read(viewers[0]!), (s) => s.width === 1280, 20000); + // Canvas capture can crop when applyConstraints changes its aspect. This + // gate checks fanout of the actual input; --display in the pool probe owns + // real display-capture geometry. + await waitForSample(async () => ({ source: await read(host), viewer: await read(viewers[0]!) }), + ({ source, viewer }) => source.width > 0 && source.height > 0 && + matchesCaptureSize(viewer.width, viewer.height, fittedCaptureSize(source, 1280, 720)), 20000); checks.liveQuality = true; stage = "pause-resume"; await click(host, 'button[aria-label="Pause sharing"]'); diff --git a/scripts/browser-local-pool-page.js b/scripts/browser-local-pool-page.js index a2ead5744..c5f66e524 100644 --- a/scripts/browser-local-pool-page.js +++ b/scripts/browser-local-pool-page.js @@ -1,7 +1,7 @@ // Actual HostPeer/BrowserEncodingPool comparison; synthetic source and local receivers only. import { BrowserEncodingPool as ProbePool } from '../src/client/media/browser-encoding-pool'; import { HostPeer as ProbeHostPeer } from '../src/client/webrtc/host-peer'; -import { applyVideoCaptureProfile as probeApplyProfile } from '../src/client/media/quality'; +import { applyVideoCaptureProfile as probeApplyProfile, QUALITY_RESOLUTIONS } from '../src/client/media/quality'; import { browserDebugEnabled, exportBrowserDebug } from '../src/client/lib/debug'; const settings = new URLSearchParams(location.search); @@ -15,8 +15,9 @@ const relay = settings.get('relay') === '1'; const shortPulse = settings.get('shortPulse') === '1'; const displaySource = settings.get('displaySource') === '1'; const quietStart = settings.get('quietStart') === '1'; -const selectedProfile = () => ({ resolution: high ? '1080p' : '480p', maxFramerate: 30, - maxBitrate: high ? 5000000 : 1700000, +const customSize = settings.has('width') ? { width: Number(settings.get('width')), height: Number(settings.get('height')) } : null; +const selectedProfile = () => ({ resolution: customSize ?? QUALITY_RESOLUTIONS[high ? '1080p' : '480p'], maxFramerate: 30, + maxBitrate: customSize ? 12000000 : high ? 5000000 : 1700000, degradationPreference: settings.get('preference') || (high ? 'maintain-resolution' : 'balanced'), maxAudioBitrate: Number(settings.get('audioBitrate') || 128000) }); const productPool = mode === 'carrier' ? new ProbePool() : null; @@ -46,6 +47,8 @@ const poolState = () => productPool ? { ready: g.ready, failed: g.failed, output: g.output })) } : null; window.probeProgress = () => ({ phase, sourceFrame, errors, pool: poolState(), + sourceSize: { width: rawSource?.getSettings().width, height: rawSource?.getSettings().height }, + captured: sourceFrames.slice(-3), decoded: monitors.map(m => ({ role: m.role, last: m.samples.at(-1) })), peers: peers.map(p => ({ connection: p.connectionState, ice: p.iceConnectionState })), audio: audioMonitors.map(m => ({ role: m.role, attached: m.attached, pulses: m.pulses.length })) }); @@ -231,10 +234,10 @@ async function run() { audioTrack = destination.stream.getAudioTracks()[0]; tracks.push(audioTrack); } const source = document.createElement('canvas'); - source.width = high ? 1920 : 640; source.height = high ? 1080 : 360; + source.width = customSize?.width ?? (high ? 1920 : 640); source.height = customSize?.height ?? (high ? 1080 : 360); if (!nativeSource) document.body.prepend(source); const ctx = source.getContext('2d'); - if (high) ctx.scale(3, 3); + ctx.scale(source.width / 640, source.height / 360); if (displaySource) { document.body.style.margin = '0'; source.style.cssText = 'position:fixed;inset:0;width:100vw;height:100vh;z-index:10'; @@ -294,8 +297,10 @@ async function run() { let activated = null; for (;;) { const members = [...productPool.members]; + // Compatible allocations share an owner; distinct BWE budgets must + // remain independent even when this synthetic scene encodes cheaply. const sharing = members.length === productPeers.length && members.every(m => m.current && m.carrier && !m.replacement) && - new Set(members.map(m => m.current)).size === 1; + new Set(members.map(m => m.current)).size === new Set(members.map(m => m.budget)).size; if (sharing) activated ??= performance.now(); else activated = null; if (activated !== null && monitors.every(m => { @@ -335,16 +340,16 @@ async function run() { phase = name; await wait(duration); snapshots.push(await snapshot()); if (name.startsWith('quiet-') || name.startsWith('paused-')) return; for (const monitor of monitors) { - const last = monitor.samples.at(-1), ceiling = { '480p': 480, '720p': 720, '1080p': 1080 }[appliedProfile.resolution]; + const last = monitor.samples.at(-1), ceiling = appliedProfile.resolution; if (last?.phase !== name || (!displaySource && (last.age === null || last.age >= 1000)) || - performance.now() - last.at >= 1000 || last.height > ceiling) + performance.now() - last.at >= 1000 || last.width > ceiling.width || last.height > ceiling.height) throw Error(name + ': receiver did not deliver current frames within the requested ceiling'); } }; - const p60 = { ...selectedProfile(), resolution: '720p', maxFramerate: 60, maxBitrate: 5000000 }; + const p60 = { ...selectedProfile(), resolution: QUALITY_RESOLUTIONS['720p'], maxFramerate: 60, maxBitrate: 5000000 }; phase = 'profile-720p60'; await apply(p60); await observe(phase); producing = false; phase = 'quiet-profile-480p30'; - await apply({ ...p60, resolution: '480p', maxFramerate: 30, maxBitrate: 3000000 }); + await apply({ ...p60, resolution: QUALITY_RESOLUTIONS['480p'], maxFramerate: 30, maxBitrate: 3000000 }); await observe(phase, 2500); producing = true; await observe('resumed-480p30'); rawSource.enabled = false; productPeers.forEach(peer => peer.setPaused(true)); diff --git a/scripts/browser-local-pool-probe.ts b/scripts/browser-local-pool-probe.ts index 132a6b298..902d8cf34 100644 --- a/scripts/browser-local-pool-probe.ts +++ b/scripts/browser-local-pool-probe.ts @@ -6,15 +6,18 @@ import { join, resolve } from 'node:path'; import { tmpdir } from 'node:os'; import { CdpConnection, cleanupRun, createPage, evaluate, launchChrome, reservePort, waitForSample, waitForVersion } from './browser-gate-harness'; import { createPoolShaper } from './browser-pool-shaper.mjs'; +import { parseVideoGateSize } from './media-gate-geometry'; const mode = process.argv[2] || 'carrier', args = process.argv.slice(3); const flags = new Set(['--product', '--1080', '--h264', '--single', '--av', '--late', '--background', '--native-source', '--lifecycle', '--network', '--auto', '--relay', '--debug', '--short-pulse', '--display', '--quiet-start']); if (!['ordinary', 'carrier'].includes(mode)) throw Error('Expected ordinary|carrier'); -for (const arg of args) if (!flags.has(arg) && !arg.startsWith('--rate=') && !arg.startsWith('--preference=') && !arg.startsWith('--audio-bitrate=')) +for (const arg of args) if (!flags.has(arg) && !arg.startsWith('--rate=') && !arg.startsWith('--preference=') && !arg.startsWith('--audio-bitrate=') && !arg.startsWith('--size=')) throw Error('Unsupported option: ' + arg); const codec = args.includes('--h264') ? 'H264' : 'VP8', network = args.includes('--network'); const automatic = args.includes('--auto'), high = args.includes('--1080'), av = args.includes('--av'); +const sizeArgument = args.find(arg => arg.startsWith('--size=')); +const size = sizeArgument ? parseVideoGateSize(sizeArgument.slice('--size='.length)) : undefined; const single = args.includes('--single'), late = args.includes('--late'), background = args.includes('--background'); const displaySource = args.includes('--display'); const quietStart = args.includes('--quiet-start'); @@ -24,7 +27,7 @@ const shortPulse = args.includes('--short-pulse'); const rate = Number(args.find(arg => arg.startsWith('--rate='))?.split('=')[1] || 120000); const audioBitrate = Number(args.find(arg => arg.startsWith('--audio-bitrate='))?.split('=')[1] || 128000); if (!Number.isInteger(audioBitrate) || audioBitrate < 6000 || audioBitrate > 320000) throw Error('Audio bitrate must be 6000..320000 bps'); -const preference = args.find(arg => arg.startsWith('--preference='))?.split('=')[1] || (high ? 'maintain-resolution' : 'balanced'); +const preference = args.find(arg => arg.startsWith('--preference='))?.split('=')[1] || (high || size ? 'maintain-resolution' : 'balanced'); if (!['balanced', 'maintain-resolution', 'maintain-framerate'].includes(preference)) throw Error('Unsupported degradation preference'); if (!Number.isInteger(rate) || rate <= 0) throw Error('Rate must be a positive integer in bps'); if (lifecycle && (nativeSource || network || relay)) throw Error('Lifecycle uses direct canvas/display capture without network shaping'); @@ -76,7 +79,7 @@ const port = (server.address() as { port: number }).port, debugPort = await rese const profile = await mkdtemp(join(tmpdir(), 'piik-client-media-')); const chrome = await launchChrome(process.env.CHROME_PATH || 'C:/Program Files/Google/Chrome/Application/chrome.exe', debugPort, profile, ['--headless=new', '--no-first-run', '--autoplay-policy=no-user-gesture-required', - ...(displaySource ? ['--window-size=1920,1200', '--force-device-scale-factor=1', + ...(displaySource ? [`--window-size=${size?.width ?? 1920},${size?.height ?? 1200}`, '--force-device-scale-factor=1', '--auto-accept-this-tab-capture', '--auto-select-tab-capture-source-by-title=Browser local pool'] : []), ...(background ? [] : ['--disable-background-timer-throttling', '--disable-renderer-backgrounding']), ...(nativeSource ? ['--use-fake-device-for-media-stream', '--use-fake-ui-for-media-stream'] : [])]); @@ -88,6 +91,9 @@ try { cdp = await CdpConnection.connect(version.webSocketDebuggerUrl, Date.now() + 5000); page = await createPage(cdp, 'about:blank'); const query = new URLSearchParams({ mode, codec, rate: String(rate), preference, audioBitrate: String(audioBitrate) }); + if (size) { + query.set('width', String(size.width)); query.set('height', String(size.height)); + } if (process.argv.includes('--debug')) query.set('debug', '1'); for (const [key, value] of Object.entries({ network, automatic, high, av, single, late, background, nativeSource, displaySource, quietStart, lifecycle, relay, shortPulse })) query.set(key, value ? '1' : '0'); diff --git a/scripts/media-gate-geometry.ts b/scripts/media-gate-geometry.ts index 1d388ee5e..77b04a216 100644 --- a/scripts/media-gate-geometry.ts +++ b/scripts/media-gate-geometry.ts @@ -1,3 +1,14 @@ +import { captureFramerateCeiling } from "../src/client/media/quality"; + +export function parseVideoGateSize(value: string) { + const match = /^(\d+)x(\d+)$/.exec(value); + const size = { width: Number(match?.[1]), height: Number(match?.[2]) }; + if (captureFramerateCeiling(size) < 30) { + throw new Error("Video size must be WIDTHxHEIGHT within current 30 fps capture limits"); + } + return size; +} + // Gate expectations derive from the decoded source instead of assuming a 16:9 // canvas. Two pixels allow independent NV12 rounding at successive ceilings. export function fittedCaptureSize(source: { width: number; height: number }, width: number, height: number) { diff --git a/tests/browser-gate-harness.test.ts b/tests/browser-gate-harness.test.ts index f9919cd43..6baf43452 100644 --- a/tests/browser-gate-harness.test.ts +++ b/tests/browser-gate-harness.test.ts @@ -1,6 +1,16 @@ import { expect, it } from "vitest"; import { join } from "node:path"; import { launchChrome } from "../scripts/browser-gate-harness"; +import { parseVideoGateSize } from "../scripts/media-gate-geometry"; + +it("accepts bounded numeric gate sizes and rejects malformed or oversized sources", () => { + expect(parseVideoGateSize("3840x2160")).toEqual({ width: 3840, height: 2160 }); + expect(parseVideoGateSize("2160x3840")).toEqual({ width: 2160, height: 3840 }); + expect(parseVideoGateSize("960x600")).toEqual({ width: 960, height: 600 }); + for (const value of ["", "1920", "1920x1080x2", "4.5x4", "0x1080", "1921x1080", "3840x3840", "8192x8192"]) { + expect(() => parseVideoGateSize(value)).toThrow("30 fps capture limits"); + } +}); it("reports a missing browser executable through awaited startup", async () => { await expect(launchChrome(join(import.meta.dirname, "missing-browser.exe"), 9222, From c75a20521a2d4eae4b6a5c1fe43c5e41f5ded2b0 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:39:24 +0800 Subject: [PATCH 076/262] fix(media): isolate incompatible receiver encoding fallback --- docs/research/browser-local-encoding-pool.md | 9 +++++ docs/research/native-client-media.md | 9 +++++ docs/standards/media-quality.md | 4 ++- src/client/media/browser-encoding-pool.ts | 21 +++++++----- tests/browser-encoding-pool.test.ts | 36 ++++++++++++++++---- 5 files changed, 64 insertions(+), 15 deletions(-) diff --git a/docs/research/browser-local-encoding-pool.md b/docs/research/browser-local-encoding-pool.md index 50e3d1eff..9a253b2f3 100644 --- a/docs/research/browser-local-encoding-pool.md +++ b/docs/research/browser-local-encoding-pool.md @@ -101,6 +101,15 @@ around fallback was 271 ms for H264 and 80 ms for VP8. Source-wide fallback therefore has a measurable handoff cost, even without a resolution drop or connection failure. This does not establish low-end hardware capacity. +An unmatched negotiated codec is instead connection-local. A Chrome 154 +late-Viewer control reproduced all members losing pooling when only the new +Viewer advertised different codec parameters. Scoped fallback now preserves +the first Viewer's producer and delivers fresh pictures to both Viewers in +VP8 1080p and H264 4K controls. Unit counterfactuals cover admission and active +codec changes; producer failure retains its existing source-wide fallback. +These controls establish membership isolation, not enforcement of physical +decoder limits or the cause of field quality fluctuations. + ### Local Connection Failure Control On 2026-10-08, Windows headless Chrome 154 ran the actual `HostPeer` and pool diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index e2b15c7ef..8f770af35 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -646,6 +646,15 @@ general constraint solver or second recovery owner. Numeric limits describe ceilings, not a promise of a constant rate or resolution. A receiver's limits constrain its edge or existing local output group, not every Viewer in the room. +Receiver-limit acceptance cannot rely on SDP matching alone. Chrome 154 +advertised H.264 Level 3.1 while decoding the 4K controls above. In isolated +ordinary-sender checks, lowering the answer to Level 1.2, or adding VP8 +`max-fs=1200;max-fr=15`, did not reduce the received picture size. This is not +evidence for a physical low-capability decoder. [RFC 6184's receive-level +rules](https://www.rfc-editor.org/rfc/rfc6184.html#section-8.2.2) and actual +framework behavior therefore need separate validation before promising +automatic compatibility with smaller future capability sets. + Codec support must distinguish encoding, decoding and encoded forwarding, including profile/packetization and SFU admission. Reuse SDP codec matching; describe only Native control and forwarding limits that it cannot express, diff --git a/docs/standards/media-quality.md b/docs/standards/media-quality.md index 2c2ba8349..7b0b3300b 100644 --- a/docs/standards/media-quality.md +++ b/docs/standards/media-quality.md @@ -273,7 +273,9 @@ bitrate still follows the Host ceiling; ordinary fallback restores its picture sender limits. Preparation and applied-budget completion re-evaluate pending membership immediately; accepting a recovery frame still commits the handoff. The first eligible consumer follows the same path; unsupported APIs or failed -pooling use ordinary senders. Prepared ordinary quality candidates, Native +pooling use ordinary senders. Negotiated codec incompatibility retires only +that connection's pool membership; healthy siblings keep their producers. +Prepared ordinary quality candidates, Native ingress and Browser SFU remain independent compositions. Each ordinary Browser PeerConnection owns stock WebRTC congestion control and diff --git a/src/client/media/browser-encoding-pool.ts b/src/client/media/browser-encoding-pool.ts index de152ad5e..8736cb8f9 100644 --- a/src/client/media/browser-encoding-pool.ts +++ b/src/client/media/browser-encoding-pool.ts @@ -158,16 +158,16 @@ export class BrowserEncodingPool { if (!report) { member.budget = undefined; return; } const codec = negotiatedCodec(report); if (!codec) { - if (member.current || member.pending) this.failSource(member.source); + if (member.current || member.pending) this.fallback(member); else { const outbound = videoOutbound(report); - if (outbound?.codecId && report.get(outbound.codecId)) this.failSource(member.source); + if (outbound?.codecId && report.get(outbound.codecId)) this.fallback(member); } return; } const key = codecIdentity(codec); if (member.codecKey && member.codecKey !== key && (member.current || member.pending)) { - this.failSource(member.source); return; + this.fallback(member); return; } member.codec = codec; member.codecKey = key; member.raw = videoOutbound(report); @@ -375,14 +375,19 @@ export class BrowserEncodingPool { .finally(() => { member.replacement = undefined; this.prune(); }); } + // Negotiated compatibility belongs to one connection. Its ordinary sender + // remains available without retiring a healthy sibling's shared producer. + private fallback(member: Member): void { + if (member.disabled || member.disposed) return; + member.disabled = true; + if (member.replacement) void member.replacement.then(() => this.restore(member)); + else this.restore(member); + } + private failSource(source: MediaStreamTrack): void { if (this.failedSources.has(source)) return; this.failedSources.add(source); - for (const member of this.members) if (member.source === source) { - member.disabled = true; - if (member.replacement) void member.replacement.then(() => this.restore(member)); - else this.restore(member); - } + for (const member of this.members) if (member.source === source) this.fallback(member); } private updatePauses(): void { diff --git a/tests/browser-encoding-pool.test.ts b/tests/browser-encoding-pool.test.ts index 13209f3c3..6feacedbf 100644 --- a/tests/browser-encoding-pool.test.ts +++ b/tests/browser-encoding-pool.test.ts @@ -12,15 +12,15 @@ const producers = new Map(); let pool: BrowserEncodingPool; -function stats(row: Record): RTCStatsReport { +function stats(row: Record, negotiated: RTCRtpCodec = codec): RTCStatsReport { return new Map([ ["video", { id: "video", timestamp: Date.now(), type: "outbound-rtp", kind: "video", codecId: "codec", ...row }], - ["codec", { id: "codec", timestamp: Date.now(), type: "codec", ...codec }], + ["codec", { id: "codec", timestamp: Date.now(), type: "codec", ...negotiated }], ]) as unknown as RTCStatsReport; } function member(budget = 2_000_000) { - const demand = { budget }; + const demand = { budget, codec: { ...codec } as RTCRtpCodec }; let producerId: string | null = null; const select = vi.fn((id: string, _requestKey: () => Promise, selected: () => void, accept: (metadata: RTCEncodedVideoFrameMetadata) => boolean) => { @@ -31,14 +31,14 @@ function member(budget = 2_000_000) { }); const cancelSelection = vi.fn(() => true); const encoded = { - select, cancelSelection, push: vi.fn(), passthrough: vi.fn(), setPaused: vi.fn(), + select, cancelSelection, push: vi.fn(), passthrough: vi.fn(() => { producerId = null; }), setPaused: vi.fn(), snapshot: () => ({ frames: producerId ? 30 : 0, width: 1920, height: 1080, timestamp: Date.now(), lastProducerId: producerId }), } as unknown as BrowserEncodingOutput; - const connection = { connectionState: "connected", getStats: async () => stats({ targetBitrate: demand.budget }) }; + const connection = { connectionState: "connected", getStats: async () => stats({ targetBitrate: demand.budget }, demand.codec) }; const fatal = vi.fn(); const handle = pool.create(source, {} as RTCRtpSender, connection as RTCPeerConnection, profile, encoded, async () => true, async () => undefined, fatal)!; - return { demand, select, cancelSelection, fatal, handle, producerId: () => producerId }; + return { demand, select, cancelSelection, fatal, handle, encoded, producerId: () => producerId }; } async function sample(count = 1) { @@ -86,6 +86,30 @@ afterEach(() => { }); describe("Browser encoding pool ownership", () => { + it.each(["joining", "active", "changed"])("isolates codec fallback for a %s member from healthy siblings", async (state) => { + const healthy = member(); + let limited = state === "joining" ? null : member(); + await sample(4); + const producer = [...producers.keys()][0]!; + limited ??= member(); + if (state === "changed") { + limited.demand.codec = { mimeType: "video/H264", clockRate: 90_000, + sdpFmtpLine: "packetization-mode=1;profile-level-id=42e01f" }; + vi.stubGlobal("RTCRtpSender", { getCapabilities: () => ({ codecs: [codec, limited!.demand.codec] }) }); + } else limited.demand.codec = { ...codec, sdpFmtpLine: "max-fr=15;max-fs=1200" }; + await sample(3); + + expect(limited.encoded.passthrough).toHaveBeenCalledTimes(1); + expect(limited.fatal).not.toHaveBeenCalled(); + expect(healthy.producerId()).toBe(producer.id); + expect(healthy.encoded.passthrough).not.toHaveBeenCalled(); + expect(producer.dispose).not.toHaveBeenCalled(); + const later = member(); + expect(later.handle).not.toBeNull(); + await sample(3); + expect(later.producerId()).toBe(producer.id); + }); + it("pauses a shared producer only when all of its members pause", async () => { const first = member(), second = member(); await sample(4); From 209878fbefc9ee7f0fcced42c6968a06b2100812 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:57:47 +0800 Subject: [PATCH 077/262] test(media): verify numeric sizes across SFU recovery --- docs/research/native-client-media.md | 29 ++++++++++++++++++-- docs/todo.md | 3 +- scripts/embedded-sfu-gate.ts | 41 ++++++++++++++++------------ scripts/embedded-sfu-page.ts | 25 +++++++++-------- 4 files changed, 65 insertions(+), 33 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 8f770af35..8c4621c9b 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -530,13 +530,13 @@ This is a code-derived result, not a new physical capture measurement. Evaluate orientation-independent limits and preservation of source aspect ratio together; adding numeric fields alone would retain the current geometry limitation. -The candidate first changes the existing Windows output worker: it passes the +The candidate changes the existing Windows output worker: it passes the source's fitted even dimensions to WebRTC, retaining the selected profile as a ceiling. A 640x480 source stays 640x480; portrait and ultrawide sources no longer require an encoded landscape canvas. The converter, adaptation owner, output generation and frame envelope are reused. Axis-specific limits still apply; -this does not yet give portrait capture a transposed ceiling or add a 4K setting. -Browser and macOS geometry remain separate pending work. +portrait capture can use explicit width/height ceilings but does not transpose +the selected ceiling automatically. Browser evidence follows below. Linux uses the existing [GStreamer scaler](https://gstreamer.freedesktop.org/documentation/videoconvertscale/videoscale.html) with bounded even-dimension ranges instead of fixed output dimensions. Square @@ -547,6 +547,15 @@ one pipeline, checks the output ceiling and image corners, and exercises the smallest derived output. Compilation and those regressions passed on Linux; Portal capture and hardware encoder resizing still need a physical device. +macOS remains a concrete implementation gap: ScreenCaptureKit and VideoToolbox +currently use the requested dimensions as a fixed canvas. The capture adapter +also ignores per-frame content geometry. Apple's +[capture sample](https://developer.apple.com/documentation/screencapturekit/capturing-screen-content-in-macos) +distinguishes buffer dimensions, content rectangle and pixel scale; changing only +the initial window size would not cover live resizing or Retina scale changes. +Review that boundary under the existing capture/encoder owners, then verify +portrait, small-source and resizing behavior with the macOS SDK and hardware. + Local synthetic-input controls exercised that production worker through four successive source sizes: 1080x1920, 3440x1440, 640x480 and 1920x1080. Under the 1080p ceiling, H.264 and VP8 each produced 40 frames at 606x1080, 1920x802, @@ -621,6 +630,20 @@ and `CHROME_PATH` set, then run `npm run gate:app-browser-fanout`. These local checks do not establish 4K60, physical 4K/game capture, sustained device performance, all Browser decoders or all route changes. +The existing SFU gate accepts `PIIK_EMBEDDED_SFU_VIDEO_SIZE=WIDTHxHEIGHT` for a +bounded 30 fps source; its default 720p15 control remains unchanged. Run +`npm run gate:embedded-sfu` with `PIIK_EMBEDDED_SFU_GATE=true`, +`PIIK_EMBEDDED_SFU_CODEC=h264` and `CHROME_PATH`; add +`PIIK_EMBEDDED_SFU_DISPLAY=true` for actual tab capture. Synthetic 2160x3840 and +real 3440x1440 tab sources passed decoded-picture delivery, aspect-preserving +reduction, numeric audio updates, subscription/page recovery and retirement. +The tab control also passed background capture, lower-layer demand, pause/resume +and source replacement. A separate full-UI 2160x3840 control passed SFU-to-P2P +commit and failed-P2P-to-SFU recovery through the room controller, keeping the +old SFU picture during peer preparation. These checks use loopback and synthetic +content; they do not establish physical 4K capture, every mixed Native/Browser +handoff or cross-network performance. + The proposed boundary follows the [Media Capture specification](https://www.w3.org/TR/mediacapture-streams/#constrainable-interface): requested constraints, supported capabilities and actual settings have different diff --git a/docs/todo.md b/docs/todo.md index 26e0d1449..0f6545e13 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -49,7 +49,8 @@ Implementation needs a concrete useful result within the product model. separate UI presets, numeric requests, bounded implementation capabilities and actual output. Numeric audio and video controls are implemented; the assessment owns local capture, codec, derived-output and SFU evidence. Next verify - route changes, limited receivers and cross-browser/camera orientation behavior. + mixed Native/Browser route changes, limited receivers and cross-browser/camera + orientation behavior. Browser display capture, Browser relay and Browser-to-Native fanout now have bounded high-resolution evidence. macOS geometry and physical 4K/game capture remain open. Preserve safety diff --git a/scripts/embedded-sfu-gate.ts b/scripts/embedded-sfu-gate.ts index 2da845877..8e5c4feca 100644 --- a/scripts/embedded-sfu-gate.ts +++ b/scripts/embedded-sfu-gate.ts @@ -12,7 +12,7 @@ import { import type { snapshot } from "./embedded-sfu-page"; import { readAppEndpoint } from "./app-gate-endpoint"; import { SOURCE_TITLE, sourceServer, startSourceBrowser, waitForCaptureWindow } from "./app-native-host-gate"; -import { fittedCaptureSize } from "./media-gate-geometry"; +import { fittedCaptureSize, parseVideoGateSize } from "./media-gate-geometry"; const ROOT = resolve(import.meta.dirname, ".."); const BUILD_ROOT = join(ROOT, "build", "embedded-media"); @@ -69,6 +69,9 @@ async function main(): Promise { const sfuOnly = process.env.PIIK_EMBEDDED_SFU_ONLY === "true"; const displayArm = process.env.PIIK_EMBEDDED_SFU_DISPLAY === "true"; if (displayArm && nativeArm) throw new Error("Display and Native capture are separate arms"); + const requestedSize = process.env.PIIK_EMBEDDED_SFU_VIDEO_SIZE?.trim(); + const captureSize = parseVideoGateSize(requestedSize || "1280x720"); + const sizeArgument = requestedSize ? JSON.stringify(captureSize) : "undefined"; const twoRooms = process.env.PIIK_EMBEDDED_SFU_TWO_ROOMS === "true"; if (twoRooms && !nativeArm) throw new Error("Two-room arm requires Native publication"); const codec = process.env.PIIK_EMBEDDED_SFU_CODEC?.trim() || "vp8"; @@ -95,7 +98,7 @@ async function main(): Promise { const sourceProfile = nativeArm ? await mkdtemp(join(tmpdir(), "piik-client-media-")) : null; const [sourcePort, sourceDebugPort] = nativeArm ? [await reservePort(), await reservePort()] : [0, 0]; const result = { passed: false, arm: `${nativeArm ? "native" : displayArm ? "display" : "browser"}-${codec}-simulcast${twoRooms ? "-two-rooms" : ""}`, stage: "start", - sfuOnly, stereoSeparationDb: null as number[] | null, + sfuOnly, captureSize, stereoSeparationDb: null as number[] | null, startedAt: new Date().toISOString(), finishedAt: "", processes: [] as Array<{ role: string; pid: number | null }>, high: null as { frames: number; width: number; height: number } | null, low: null as { frames: number; width: number; height: number } | null, @@ -166,10 +169,10 @@ async function main(): Promise { if (request.url === "/embedded-sfu-source") { response.setHeader("Content-Type", "text/html; charset=utf-8"); response.end(`Piik SFU captured source - `); + function draw(){ctx.fillStyle='#167d8d';ctx.fillRect(0,0,canvas.width,canvas.height);ctx.fillStyle='#f2d748'; + ctx.fillRect((n++*17)%canvas.width,60,100,100);requestAnimationFrame(draw)}draw();`); return; } if (request.url !== "/embedded-sfu-gate") return next(); @@ -229,15 +232,15 @@ async function main(): Promise { } if (displayArm) { const sourcePage = await createPage(cdp, `${origin}/embedded-sfu-source`); - await cdp.call("Emulation.setDeviceMetricsOverride", { width: 1280, height: 720, + await cdp.call("Emulation.setDeviceMetricsOverride", { ...captureSize, deviceScaleFactor: 1, mobile: false }, sourcePage.sessionId, Date.now() + 3_000); } host = await createPage(cdp, `${origin}/embedded-sfu-gate`, AUDIO_PROBE); viewer = await createPage(cdp, `${origin}/embedded-sfu-gate`, AUDIO_PROBE); result.stage = "authenticate"; const room = await call(host, nativeArm - ? `gate.startHost(${JSON.stringify({ title: SOURCE_TITLE, port: nativePort })}, ${JSON.stringify(codec)})` - : `gate.startHost(undefined, ${JSON.stringify(codec)}, ${displayArm})`); + ? `gate.startHost(${JSON.stringify({ title: SOURCE_TITLE, port: nativePort })}, ${JSON.stringify(codec)}, false, ${sizeArgument})` + : `gate.startHost(undefined, ${JSON.stringify(codec)}, ${displayArm}, ${sizeArgument})`); await until(host, (value) => value.authenticated); await call(viewer, `gate.startViewer(${JSON.stringify(room)})`); result.stage = "simulcast-first-frame"; @@ -247,12 +250,12 @@ async function main(): Promise { secondHost = await createPage(cdp, `${origin}/embedded-sfu-gate`); secondViewer = await createPage(cdp, `${origin}/embedded-sfu-gate`, AUDIO_PROBE); const secondRoom = await call(secondHost, - `gate.startHost(${JSON.stringify({ title: SOURCE_TITLE, port: nativePort })}, ${JSON.stringify(codec)})`); + `gate.startHost(${JSON.stringify({ title: SOURCE_TITLE, port: nativePort })}, ${JSON.stringify(codec)}, false, ${sizeArgument})`); if (secondRoom.roomId === room.roomId) throw new Error("Native sessions did not create independent rooms"); await until(secondHost, (value) => value.authenticated); await call(secondViewer, `gate.startViewer(${JSON.stringify(secondRoom)})`); await until(secondViewer, (value) => value.committed && value.audioKbps > 0); - await call(secondViewer, "gate.waitForFrames(1280, 720, 15, true)"); + await call(secondViewer, `gate.waitForFrames(${captureSize.width}, ${captureSize.height}, 15, true)`); await waitForSample(() => call(secondViewer!, "gate.audioEnergy()", 3_000), (energy) => { result.secondRoom!.audioBefore = energy; return energy > 0; }, 5_000); const second = await call(secondHost, "gate.snapshot()"); @@ -267,7 +270,7 @@ async function main(): Promise { : null; const sourceBefore = await sourceSample(); try { - result.high = await call(viewer, nativeArm ? "gate.waitForFrames(1280, 720, 300, true)" : "gate.waitForFrames(1280, 720)", nativeArm ? 32_000 : 25_000); + result.high = await call(viewer, `gate.waitForFrames(${captureSize.width}, ${captureSize.height}, ${nativeArm ? "300, true" : "15"})`, nativeArm ? 32_000 : 25_000); } finally { const sourceAfter = await sourceSample().catch(() => null); if (sourceBefore && sourceAfter) result.sourceDrawing = { @@ -300,26 +303,28 @@ async function main(): Promise { throw new Error("Gate must capture a separate real tab with the Host hidden"); } const hiddenStart = Date.now(); - const hidden = await call<{ frames: number }>(viewer, "gate.waitForFrames(1280, 720, 150)", 32_000); + const fullSize = `${result.high!.width}, ${result.high!.height}`; + const hidden = await call<{ frames: number }>(viewer, `gate.waitForFrames(${fullSize}, 150)`, 32_000); result.displayLifecycle = { hiddenFrames: hidden.frames, hiddenElapsedMs: Date.now() - hiddenStart, lowLayerDecoded: false, resumed: false, replaced: false }; result.mediaDiagnostics = await Promise.all([host, viewer].map(page => evaluate(cdp!, page, "window.__piikGateMediaStats()", Date.now() + 3_000))); result.stage = "display-layer-demand"; await call(host, "gate.demandLayers(1)"); - await call(viewer, "gate.waitForFrames(640, 360)"); + const half = fittedCaptureSize(result.high!, result.high!.width / 2, result.high!.height / 2); + await call(viewer, `gate.waitForFrames(${half.width}, ${half.height}, 15, false, 2)`); result.displayLifecycle.lowLayerDecoded = true; await call(host, "gate.demandLayers(2)"); - await call(viewer, "gate.waitForFrames(1280, 720)"); + await call(viewer, `gate.waitForFrames(${fullSize})`); result.stage = "display-pause-resume"; await call(host, "gate.pause(true)"); await new Promise(resolve => setTimeout(resolve, 500)); await call(host, "gate.pause(false)"); - await call(viewer, "gate.waitForFrames(1280, 720)"); + await call(viewer, `gate.waitForFrames(${fullSize})`); result.displayLifecycle.resumed = true; result.stage = "display-source-replacement"; if (!await call(host, "gate.replaceSource()")) throw new Error("Display source replacement failed"); - await call(viewer, "gate.waitForFrames(1280, 720)"); + await call(viewer, `gate.waitForFrames(${fullSize})`); result.displayLifecycle.replaced = true; } result.stage = "live-profile"; @@ -327,8 +332,8 @@ async function main(): Promise { result.host = await call(host, "gate.snapshot()"); throw new Error("Live profile update failed"); } - const lowSize = nativeArm ? fittedCaptureSize(result.high!, 854, 480) : { width: 854, height: 480 }; - result.low = await call(viewer, `gate.waitForFrames(${lowSize.width}, ${lowSize.height}, 15, false, ${nativeArm ? 2 : 0})`); + const lowSize = fittedCaptureSize(result.high!, 854, 480); + result.low = await call(viewer, `gate.waitForFrames(${lowSize.width}, ${lowSize.height}, 15, false, 2)`); await waitForSample(() => call(viewer!, "gate.audioEnergy()", 3_000), (energy) => { result.audioEnergy.low = energy; return energy > result.audioEnergy.high; }, 5_000); result.stage = "viewer-media-reload"; diff --git a/scripts/embedded-sfu-page.ts b/scripts/embedded-sfu-page.ts index 13659cded..577878715 100644 --- a/scripts/embedded-sfu-page.ts +++ b/scripts/embedded-sfu-page.ts @@ -16,12 +16,12 @@ import { NativeSfuPublisher } from "../src/client/native/native-sfu-publisher"; import { defaultNativeCapturePath } from "../src/client/native/capture-selection"; import { HostSfuRoute, type HostPublisherTransport } from "../src/client/media/host-sfu-route"; import { ViewerSfuRoute } from "../src/client/media/viewer-sfu-route"; +import { fittedCaptureSize } from "./media-gate-geometry"; const high: QualitySettings = { resolution: { width: 1280, height: 720 }, maxFramerate: 15, maxBitrate: 2_000_000, degradationPreference: "maintain-resolution", maxAudioBitrate: 64000, }; -const low: QualitySettings = { ...high, resolution: { width: 854, height: 480 } }; const shareGeneration = crypto.randomUUID(); const clientId = crypto.randomUUID(); let socket: WebSocket | null = null; @@ -193,12 +193,13 @@ async function connect(room: CreateRoomResponse, host: boolean): Promise { roomId: room.roomId, clientId } as const; send(host ? { ...common, role: "host", token: room.hostToken, shareGeneration, - qualitySettings: high, routePolicy: { peerOnly: false, topologyOptimization: false, natPrediction: false } } + qualitySettings: profile, routePolicy: { peerOnly: false, topologyOptimization: false, natPrediction: false } } : { ...common, role: "viewer", viewerGrant: new URLSearchParams(new URL(room.inviteUrl).hash.slice(1)).get("v")! }); } -export async function startHost(nativeSource?: { title: string; port: number }, selectedCodec: "vp8" | "h264" = "vp8", display = false): Promise { +export async function startHost(nativeSource?: { title: string; port: number }, selectedCodec: "vp8" | "h264" = "vp8", display = false, size?: QualitySettings["resolution"]): Promise { codec = selectedCodec; + profile = size ? { ...high, resolution: size, maxFramerate: 30, maxBitrate: 8_000_000 } : high; const response = await fetch("/api/rooms", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ codeEntryPolicy: "private" }), @@ -215,7 +216,7 @@ export async function startHost(nativeSource?: { title: string; port: number }, if (!stopping && event.type === "share-ended" && event.shareId === shareGeneration) failed("Native capture ended"); }); const started = await native.startShare({ shareId: shareGeneration, source: target, - codec, audio: true, profile: high, edgeCapacity: 2, ...path }); + codec, audio: true, profile, edgeCapacity: 2, ...path }); nativeShareStarted = true; source = new MediaStream(); if (started.codec !== codec || !started.audio) throw new Error("Requested Native codec and Opus capture did not start"); @@ -223,8 +224,8 @@ export async function startHost(nativeSource?: { title: string; port: number }, return room; } canvas = document.createElement("canvas"); - canvas.width = 1280; - canvas.height = 720; + canvas.width = profile.resolution.width; + canvas.height = profile.resolution.height; canvas.style.cssText = "max-width:100%;width:640px"; document.body.append(canvas); const context = canvas.getContext("2d")!; @@ -237,10 +238,10 @@ export async function startHost(nativeSource?: { title: string; port: number }, context.fillStyle = "#ffffff"; context.font = "48px sans-serif"; context.fillText(String(frame), 40, 240); - }, 1000 / 15); + }, 1000 / profile.maxFramerate); source = display ? await navigator.mediaDevices.getDisplayMedia({ - video: { width: 1280, height: 720, frameRate: 15 }, audio: false, - }) : canvas.captureStream(15); + video: { ...profile.resolution, frameRate: profile.maxFramerate }, audio: false, + }) : canvas.captureStream(profile.maxFramerate); audio = new AudioContext(); const gain = audio.createGain(); gain.gain.value = 0.05; @@ -377,11 +378,13 @@ export async function waitForFrames(width: number, height: number, minimumFrames export async function lowerProfile(): Promise { if (!publisher) throw new Error("Publisher is unavailable"); + const low = { ...profile, resolution: { width: 854, height: 480 } }; if (native) await native.updateShare(shareGeneration, low); else { if (!canvas) throw new Error("Browser source is unavailable"); - canvas.width = 854; - canvas.height = 480; + const size = fittedCaptureSize(canvas, low.resolution.width, low.resolution.height); + canvas.width = size.width; + canvas.height = size.height; } profile = low; send({ type: "set-quality-settings", qualitySettings: low }); From b9771b5e6a86cef54a3fadda60d1a54c9ae0f7ca Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:27:30 +0800 Subject: [PATCH 078/262] docs(transport): record bounded address-handoff evidence --- docs/research/nat-traversal.md | 21 +++++++++++++++++++++ docs/todo.md | 3 +++ 2 files changed, 24 insertions(+) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 7a0cba49f..c8caaee01 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -635,6 +635,27 @@ reachable class. The control makes no explicit rediscovery call; its passing result validates the measurement and available-media baseline, not universal direct recovery. It does not run Piik's complete reconnect schedule. +An actual local-address-change control on 2026-10-09 replaces one endpoint's +IPv4 address, default route and public UDP mapping, allowing upstream network +monitoring to react. Discovery stays available, without an explicit endpoint +update or diagnostic Ping. With Piik's unchanged Viewer recovery owner and +Trickle exchange, three repetitions per direction preserve Tailcat's same media +session: direct delivery returns in 1.25–6.40 seconds with no owner restart. +Both default inner checks and the isolated 64-check candidate pass this control. +The ordinary Pion comparison requests one ICE restart and returns direct media +in 9.14–9.24 seconds. Relay accounting separates first available media from +direct delivery; another two seconds of fresh SRTP after DERP closes confirms it. + +The actual Native Engine's wildcard UDP socket is a separate baseline: both +directions stop delivering after the address change without new SDP, and fresh +Native discovery restores local playback on that same changed network. This +avoids attributing only Pion's default per-interface socket behavior to Piik. +These controls establish a bounded address-handoff benefit, not an additional +initial NAT-reachability class, physical Wi-Fi roaming or complete App/decoder +acceptance. They do not resolve the gateway-only or prolonged-outage controls +above. Supported dependencies, room grants and representative network evidence +remain prerequisites for product adoption. + Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof required fresh bidirectional delivery after DERP closed. One side reporting a diff --git a/docs/todo.md b/docs/todo.md index 0f6545e13..e06200658 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -69,6 +69,9 @@ Implementation needs a concrete useful result within the product model. [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) have not established improved direct reachability. Seek a repeatable benefit before accepting room-scoped peer keys, flow admission and relay operation. + Isolated local-address changes now preserve the same session under the actual + Viewer recovery owner. Verify that bounded benefit through the complete App + pipeline and physical networks; it does not establish another reachable NAT class. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) and RTCP/MTU boundaries, retained-flow recovery beyond the passing serial From bef002ed556a50a257acaf74ab595f73ee998cf3 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:02:37 +0800 Subject: [PATCH 079/262] fix(media): preserve camera framing across quality controls --- docs/research/native-client-media.md | 23 ++++++++++++++++++++--- docs/standards/media-quality.md | 8 ++++++-- docs/todo.md | 6 +++--- src/client/media/camera-previews.ts | 8 +++----- src/client/media/capture-devices.ts | 11 +++++++++++ src/client/media/quality.ts | 14 ++++++-------- tests/camera-previews.test.ts | 1 + tests/host-audio.test.ts | 2 ++ tests/host-peer-tracks.test.ts | 4 ++-- tests/quality.test.ts | 13 +++++++------ tests/sender-video-track.test.ts | 2 +- 11 files changed, 62 insertions(+), 30 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 8c4621c9b..2969d94af 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -640,9 +640,26 @@ reduction, numeric audio updates, subscription/page recovery and retirement. The tab control also passed background capture, lower-layer demand, pause/resume and source replacement. A separate full-UI 2160x3840 control passed SFU-to-P2P commit and failed-P2P-to-SFU recovery through the room controller, keeping the -old SFU picture during peer preparation. These checks use loopback and synthetic -content; they do not establish physical 4K capture, every mixed Native/Browser -handoff or cross-network performance. +old SFU picture during peer preparation. The mixed Browser Host/Native Viewer +control also passed those transitions. With clarity priority, its live +2160x3840-to-360x640 edit retained the Native connection and playback track; +SFU recovery retained that size and then restored 2160x3840. Firefox 155 received +the same VP8 sizes from Chrome, retaining its track through both edits. All +controls retired their receiving connections on stop. Balanced priority may +deliver smaller adapted pictures. These local checks do not establish physical +4K capture, every mixed handoff or cross-network performance. + +Chrome 154 camera controls exposed a separate constraint bug: lowering a +540x960 input to the 1280x720 ceiling produced a cropped 540x720 picture when +both axes also carried `ideal` values. Maximum-only constraints produced +405x720, retained portrait shape through a subsequent 360x640 setting, and +restored 540x960. Camera startup and picker requests could likewise crop to their +ideal axes; they now share an uncropped device-format request (`resizeMode: +{ exact: "none" }`). Portrait thumbnails then measured 180x320 instead of 320x180. +A landscape control retained shape within rounding. Shared live constraints use +ceilings without a fixed aspect ratio or rotation listener, following the Media +Capture distinction below. Synthetic devices do not establish physical phone +orientation behavior. The proposed boundary follows the [Media Capture specification](https://www.w3.org/TR/mediacapture-streams/#constrainable-interface): diff --git a/docs/standards/media-quality.md b/docs/standards/media-quality.md index 7b0b3300b..8a2eba030 100644 --- a/docs/standards/media-quality.md +++ b/docs/standards/media-quality.md @@ -18,7 +18,9 @@ contract. [Status](../status.md) indexes delivery and remaining acceptance. latter uses one supported native codec path and never infers a target from a title. An ordinary Web Host does not probe localhost. - Camera is a peer source choice in the same picker, including on phones. Only - the chosen source requests permission. Browser/camera Host microphone capture + the chosen source requests permission. Camera input and thumbnails request + an uncropped device format; sender constraints own later scaling. + Browser/camera Host microphone capture is explicit and defaults to Browser voice processing, separately from screen audio. The Host may turn that processing off for music or virtual audio inputs; the input change preserves the existing mixed output and leaves the previous @@ -81,7 +83,9 @@ contract. [Status](../status.md) indexes delivery and remaining acceptance. not intentionally stop capture, encoding, or upload. - Live quality changes update the current capture and Host sender-track constraints plus sender ceilings without reopening source selection or - replacing a healthy route. + replacing a healthy route. Shared Browser capture constraints express size + ceilings, not ideal dimensions or a fixed aspect ratio: lowering a ceiling + must not request a crop, and source orientation remains Browser-owned. - Native quality changes use the same room settings. The App prepares a new platform capture/encoder generation, then replaces the old generation behind the same encoded source and PeerConnections; audio-only changes update the diff --git a/docs/todo.md b/docs/todo.md index e06200658..210f0a86b 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -48,9 +48,9 @@ Implementation needs a concrete useful result within the product model. [media assessment](./research/native-client-media.md#media-capability-extension-assessment): separate UI presets, numeric requests, bounded implementation capabilities and actual output. Numeric audio and video controls are implemented; the assessment - owns local capture, codec, derived-output and SFU evidence. Next verify - mixed Native/Browser route changes, limited receivers and cross-browser/camera - orientation behavior. + owns local capture, codec, derived-output, SFU and bounded mixed-route evidence. + Extend Browser interoperability and mixed-topology checks to limited receivers + and physical camera orientation behavior. Browser display capture, Browser relay and Browser-to-Native fanout now have bounded high-resolution evidence. macOS geometry and physical 4K/game capture remain open. Preserve safety diff --git a/src/client/media/camera-previews.ts b/src/client/media/camera-previews.ts index fd2804cfc..cceb177cd 100644 --- a/src/client/media/camera-previews.ts +++ b/src/client/media/camera-previews.ts @@ -1,4 +1,4 @@ -import { browserCameras, type CaptureDevice } from "./capture-devices"; +import { browserCameras, cameraCaptureConstraints, type CaptureDevice } from "./capture-devices"; export type CameraPreview = CaptureDevice & { preview: string | null }; @@ -47,10 +47,8 @@ export async function loadCameraPreviews(signal: AbortSignal, const release = () => { owned?.getTracks().forEach(track => track.stop()); owned = null; }; const open = async (id: string) => { signal.throwIfAborted(); - owned = await navigator.mediaDevices.getUserMedia({ audio: false, video: { - ...(id ? { deviceId: { exact: id } } : { facingMode: { ideal: "environment" } }), - width: { ideal: 320 }, height: { ideal: 180 }, frameRate: { ideal: 15, max: 15 }, - } }); + owned = await navigator.mediaDevices.getUserMedia({ audio: false, + video: cameraCaptureConstraints({ width: 320, height: 180 }, 15, id) }); // A permission promise can settle after the picker has closed. if (signal.aborted) { release(); signal.throwIfAborted(); } return owned; diff --git a/src/client/media/capture-devices.ts b/src/client/media/capture-devices.ts index 922ea4caf..0ec10f214 100644 --- a/src/client/media/capture-devices.ts +++ b/src/client/media/capture-devices.ts @@ -1,5 +1,16 @@ export type CaptureDevice = { id: string; label: string }; +/** Choose a camera format without cropping; sender constraints own later scaling. */ +export function cameraCaptureConstraints(size: { width: number; height: number }, + maxFramerate: number, deviceId = ""): MediaTrackConstraints & { resizeMode: ConstrainDOMString } { + return { + ...(deviceId ? { deviceId: { exact: deviceId } } : { facingMode: { ideal: "environment" } }), + resizeMode: { exact: "none" }, + width: { ideal: size.width }, height: { ideal: size.height }, + frameRate: { ideal: maxFramerate, max: maxFramerate }, + }; +} + // Enumeration never requests permission or opens a device. Device IDs remain // local; the capture owner applies an explicit choice when permission is given. export async function browserCaptureDevices(kind: "audioinput" | "videoinput"): Promise { diff --git a/src/client/media/quality.ts b/src/client/media/quality.ts index 6f521c751..6482c1840 100644 --- a/src/client/media/quality.ts +++ b/src/client/media/quality.ts @@ -12,6 +12,7 @@ import { displayMediaOptions } from "./audio-capture"; import { browserDebugEnabled, debugOperation } from "../lib/debug"; import { debugTrack } from "../lib/debug-webrtc"; import { applySenderCaptureConstraints, senderCaptureTrack } from "./sender-video-track"; +import { cameraCaptureConstraints } from "./capture-devices"; export type { DegradationPreference, @@ -212,8 +213,9 @@ export function qualitySettingsLabel(settings: QualitySettings): string { function captureConstraints(profile: QualityProfile): MediaTrackConstraints { const resolution = profile.resolution; return { - width: { ideal: resolution.width, max: resolution.width }, - height: { ideal: resolution.height, max: resolution.height }, + // A ceiling must not ask the Browser to fill both axes by cropping a source. + width: { max: resolution.width }, + height: { max: resolution.height }, frameRate: { ideal: profile.maxFramerate, max: profile.maxFramerate, @@ -259,12 +261,8 @@ export type BrowserCaptureSource = "browser" | "camera"; export async function captureBrowserSource(profile: QualityProfile, source: BrowserCaptureSource, deviceId = ""): Promise { if (source !== "camera") return captureDisplay(profile); - const resolution = profile.resolution; - const stream = await navigator.mediaDevices.getUserMedia({ audio: false, video: { - ...(deviceId ? { deviceId: { exact: deviceId } } : { facingMode: { ideal: "environment" } }), - width: { ideal: resolution.width }, height: { ideal: resolution.height }, - frameRate: { ideal: profile.maxFramerate, max: profile.maxFramerate }, - } }); + const stream = await navigator.mediaDevices.getUserMedia({ audio: false, + video: cameraCaptureConstraints(profile.resolution, profile.maxFramerate, deviceId) }); const track = stream.getVideoTracks()[0]; if (!track || track.readyState === "ended") { stream.getTracks().forEach((track) => track.stop()); diff --git a/tests/camera-previews.test.ts b/tests/camera-previews.test.ts index b2019ab4f..b4d68441a 100644 --- a/tests/camera-previews.test.ts +++ b/tests/camera-previews.test.ts @@ -14,6 +14,7 @@ function setup({ permission = true, frames = true } = {}) { }; const getUserMedia = vi.fn(async (options: MediaStreamConstraints) => { expect(options.audio).toBe(false); + expect(options.video).toHaveProperty("resizeMode", { exact: "none" }); expect(tracks.filter(track => track.readyState === "live")).toHaveLength(0); permission = true; const id = (options.video as MediaTrackConstraints).deviceId as ConstrainDOMStringParameters | undefined; diff --git a/tests/host-audio.test.ts b/tests/host-audio.test.ts index 1daacc9bd..4112bdeed 100644 --- a/tests/host-audio.test.ts +++ b/tests/host-audio.test.ts @@ -52,6 +52,7 @@ test("camera is a bounded video source and never implicitly requests a microphon getUserMedia.mockResolvedValue(source); expect(await captureBrowserSource(QUALITY_PROFILES["1080p30"], "camera")).toBe(source); expect(getUserMedia).toHaveBeenCalledWith({ audio: false, video: { + resizeMode: { exact: "none" }, facingMode: { ideal: "environment" }, width: { ideal: 1920 }, height: { ideal: 1080 }, frameRate: { ideal: 30, max: 30 }, } }); @@ -187,6 +188,7 @@ test("camera selection requests one exact device without microphone permission", getUserMedia.mockResolvedValue(media(new Track("video"))); await captureBrowserSource(QUALITY_PROFILES["1080p30"], "camera", "front-camera"); expect(getUserMedia).toHaveBeenCalledWith({ audio: false, video: expect.objectContaining({ deviceId: { exact: "front-camera" } }) }); + expect(getUserMedia.mock.calls[0][0].video).toHaveProperty("resizeMode", { exact: "none" }); expect(getUserMedia.mock.calls[0][0].video).not.toHaveProperty("facingMode"); }); diff --git a/tests/host-peer-tracks.test.ts b/tests/host-peer-tracks.test.ts index 0d367ace6..f1f070b73 100644 --- a/tests/host-peer-tracks.test.ts +++ b/tests/host-peer-tracks.test.ts @@ -1140,8 +1140,8 @@ describe("HostPeer source replacement", () => { peer.updateCaptureProfile(QUALITY_PROFILES["1080p60"]), ).resolves.toBe(true); expect(senderVideo.applyConstraints).toHaveBeenLastCalledWith({ - width: { ideal: 1920, max: 1920 }, - height: { ideal: 1080, max: 1080 }, + width: { max: 1920 }, + height: { max: 1080 }, frameRate: { ideal: 60, max: 60 }, }); expect(sourceVideo.applyConstraints).not.toHaveBeenCalled(); diff --git a/tests/quality.test.ts b/tests/quality.test.ts index a65ab7b37..b08fb7dd5 100644 --- a/tests/quality.test.ts +++ b/tests/quality.test.ts @@ -105,8 +105,8 @@ describe("realtime quality controls", () => { expect(getDisplayMedia).toHaveBeenCalledWith({ video: { - width: { ideal: 1920, max: 1920 }, - height: { ideal: 1080, max: 1080 }, + width: { max: 1920 }, + height: { max: 1080 }, frameRate: { ideal: 30, max: 30 }, }, audio: { @@ -123,19 +123,20 @@ describe("realtime quality controls", () => { expect(audioTrack.contentHint).toBe("music"); }); - it("changes custom capture ceilings without selecting the source again", async () => { + it("changes capture ceilings without requesting a crop or locking source orientation", async () => { const { stream, videoTrack } = createVideoStream(); + videoTrack.getSettings = () => ({ width: 540, height: 960, aspectRatio: 9 / 16 }); await applyCaptureProfile(stream, { - resolution: { width: 1920, height: 1200 }, + resolution: { width: 1280, height: 720 }, maxFramerate: 45, maxBitrate: 9_500_000, degradationPreference: "balanced", }); expect(videoTrack.applyConstraints).toHaveBeenCalledWith({ - width: { ideal: 1920, max: 1920 }, - height: { ideal: 1200, max: 1200 }, + width: { max: 1280 }, + height: { max: 720 }, frameRate: { ideal: 45, max: 45 }, }); }); diff --git a/tests/sender-video-track.test.ts b/tests/sender-video-track.test.ts index 0fa005d97..6c60e7c53 100644 --- a/tests/sender-video-track.test.ts +++ b/tests/sender-video-track.test.ts @@ -79,7 +79,7 @@ describe("sender display-frame ownership", () => { expect(p.write).toHaveBeenCalledWith(frame); expect(frame.close).toHaveBeenCalledOnce(); await applyVideoCaptureProfile(track, QUALITY_PROFILES["720p30"]); - expect(input.applyConstraints).toHaveBeenCalledWith(expect.objectContaining({ width: { ideal: 1280, max: 1280 } })); + expect(input.applyConstraints).toHaveBeenCalledWith(expect.objectContaining({ width: { max: 1280 } })); expect(input.applyConstraints).toHaveBeenLastCalledWith(expect.objectContaining({ frameRate: { min: 1, ideal: 30, max: 30 } })); expect((track as unknown as Track).applyConstraints).not.toHaveBeenCalled(); From 2f111daa9d88c36e96580e0829a85644461fe760 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:37:50 +0800 Subject: [PATCH 080/262] docs(research): record Native tunnel continuity limits --- docs/research/nat-traversal.md | 22 ++++++++++++++++++++++ docs/todo.md | 3 +++ 2 files changed, 25 insertions(+) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index c8caaee01..805c25e41 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -656,6 +656,28 @@ acceptance. They do not resolve the gateway-only or prolonged-outage controls above. Supported dependencies, room grants and representative network evidence remain prerequisites for product adoption. +The 2026-10-09 remote-socket overlay extends that comparison through the actual +Native encoded source, sender, receiver and ordinary local playback. It leaves +product files untouched and admits one fixture peer flow per engine. Unchanged +ordinary/Tailcat controls pass, and both handoff directions can retain media +after DERP closure. Stronger fresh-content checks expose a limit: two of three +sender-handoff repetitions pause output for roughly two seconds while ICE stays +connected; one remains paused at the fifteen-second observation boundary. +The outgoing frame counter also stops as the sender estimate falls to about +20 kbps. This locates those gaps at forwarding/adaptation, rather than proving +that the underlay lost the connection. + +The fixed 8×8 input deliberately has no live rate-controllable encoder. Declaring +it rate-controlled removes the later forwarding pauses in three repetitions, +but all three still miss the strict first fresh-content window and later deliver +about 34–37 frames per second instead of the requested 50. This is an ablation, +not a valid capture implementation or a proposed product switch. Actual Native +capture already declares its live lowest output controllable and applies codec +budgets; raw forwarding retains the framework's pause policy. Full App acceptance +must include that encoder owner, local Browser decode and recovery, and useful +picture continuity. A preserved ICE session alone does not establish a net +handoff benefit. No production adaptation rule or timeout changed. + Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof required fresh bidirectional delivery after DERP closed. One side reporting a diff --git a/docs/todo.md b/docs/todo.md index 210f0a86b..b8a620a31 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -72,6 +72,9 @@ Implementation needs a concrete useful result within the product model. Isolated local-address changes now preserve the same session under the actual Viewer recovery owner. Verify that bounded benefit through the complete App pipeline and physical networks; it does not establish another reachable NAT class. + The actual Native media overlay can retain the transport but its fixed-input + continuity controls expose forwarding pauses. Include the live encoder and + decoded picture continuity; do not treat the rate-control ablation as a fix. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) and RTCP/MTU boundaries, retained-flow recovery beyond the passing serial From 31f8297a1d32972e04a87e80dd4653e0edf8e313 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:22:05 +0800 Subject: [PATCH 081/262] fix(media): serialize forwarding probe padding once --- internal/media/forwarding/transport.go | 15 +++++++++++++++ internal/media/forwarding/transport_test.go | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/internal/media/forwarding/transport.go b/internal/media/forwarding/transport.go index af7d71ead..0ab8a5474 100644 --- a/internal/media/forwarding/transport.go +++ b/internal/media/forwarding/transport.go @@ -636,6 +636,9 @@ func (ticket *pacedPacket) WriteRTP(*rtp.Header, []byte) (int, error) { if packet == nil { return 0, io.ErrClosedPipe } + if packet.Header.Padding { + packet.WriteStream = pionPaddingWriter{packet.WriteStream} + } written, err := queue.sender.SendPacket(packet) if err == nil && written > 0 { queue.packets.Add(1) @@ -647,6 +650,18 @@ func (ticket *pacedPacket) WriteRTP(*rtp.Header, []byte) (int, error) { func (*pacedPacket) Write([]byte) (int, error) { return 0, io.ErrClosedPipe } +// LiveKit's padding payload already includes its trailing padding bytes. Keep +// that size for pacing/BWE, but let Pion serialize the padding exactly once. +type pionPaddingWriter struct{ webrtc.TrackLocalWriter } + +func (writer pionPaddingWriter) WriteRTP(header *rtp.Header, payload []byte) (int, error) { + padding := int(header.PaddingSize) + if padding == 0 || padding > len(payload) { + return 0, errors.New("invalid forwarding padding") + } + return writer.TrackLocalWriter.WriteRTP(header, payload[:len(payload)-padding]) +} + type closedPacketWriter struct{} func (closedPacketWriter) WriteRTP(*rtp.Header, []byte) (int, error) { return 0, io.ErrClosedPipe } diff --git a/internal/media/forwarding/transport_test.go b/internal/media/forwarding/transport_test.go index 2cb4eda68..59d7bda08 100644 --- a/internal/media/forwarding/transport_test.go +++ b/internal/media/forwarding/transport_test.go @@ -92,6 +92,7 @@ func TestTransportVideoFeedbackAudioAndClose(t *testing.T) { defer receiver.Close() var videoPackets, audioPackets atomic.Uint32 firstVideo := make(chan *rtp.Packet, 1) + paddingPackets := make(chan *rtp.Packet, 8) var recoveryTimestamp atomic.Uint32 recovered := make(chan struct{}, 1) var readers sync.WaitGroup @@ -103,6 +104,12 @@ func TestTransportVideoFeedbackAudioAndClose(t *testing.T) { if readErr != nil { return } + if packet.Padding { + select { + case paddingPackets <- packet: + default: + } + } if len(packet.Payload) == 0 { continue } @@ -244,6 +251,19 @@ func TestTransportVideoFeedbackAudioAndClose(t *testing.T) { if ready || transport.prober.IsRunning() { t.Fatal("transport probed before a receiver report") } + // Exercise the library's real padding packet through Pion SRTP. Probe bytes + // must be stripped by RTP decoding, not exposed to the video decoder as media. + if transport.Output.WritePaddingRTP(255, true, false) == 0 { + t.Fatal("padding fixture did not emit a probe") + } + select { + case packet := <-paddingPackets: + if len(packet.Payload) != 0 || packet.Header.PaddingSize == 0 { + t.Fatalf("padding reached the receiver as video: payload=%d padding=%d", len(packet.Payload), packet.Header.PaddingSize) + } + case <-ctx.Done(): + t.Fatal("padding packet did not arrive") + } // A muted library sender returns zero padding bytes despite a connected PC // and valid RR. Its probe must finish, and inconclusive feedback must not // promote the BWE's internal initial-capacity sentinel into a codec budget. From 420e1ff18e9d8a453c9f8961e3dae43a8e4b0707 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:36:03 +0800 Subject: [PATCH 082/262] test(media): keep one receiver feedback owner --- internal/app/mediaedge/edge_test.go | 3 --- internal/app/mediaedge/relay_network_fixture_test.go | 1 - internal/media/forwarding/publication_test.go | 1 - internal/media/forwarding/recovery_test.go | 1 - internal/media/forwarding/transport_test.go | 1 - 5 files changed, 7 deletions(-) diff --git a/internal/app/mediaedge/edge_test.go b/internal/app/mediaedge/edge_test.go index ac65c02b6..cc2424052 100644 --- a/internal/app/mediaedge/edge_test.go +++ b/internal/app/mediaedge/edge_test.go @@ -826,9 +826,6 @@ func newReceiverWithAudio(t *testing.T, includeAudio bool, bindAddress string) * } } registry := &interceptor.Registry{} - if err := webrtc.ConfigureTWCCSender(mediaEngine, registry); err != nil { - t.Fatal(err) - } if err := webrtc.RegisterDefaultInterceptors(mediaEngine, registry); err != nil { t.Fatal(err) } diff --git a/internal/app/mediaedge/relay_network_fixture_test.go b/internal/app/mediaedge/relay_network_fixture_test.go index a0e82c72f..ea8c1d550 100644 --- a/internal/app/mediaedge/relay_network_fixture_test.go +++ b/internal/app/mediaedge/relay_network_fixture_test.go @@ -124,7 +124,6 @@ func TestRelayNetworkVP8Fixture(t *testing.T) { check(media.RegisterDefaultCodecs()) registry := &interceptor.Registry{} check(webrtc.RegisterDefaultInterceptors(media, registry)) - check(webrtc.ConfigureTWCCSender(media, registry)) receiver, createErr := webrtc.NewAPI(webrtc.WithMediaEngine(media), webrtc.WithInterceptorRegistry(registry), webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) check(createErr) diff --git a/internal/media/forwarding/publication_test.go b/internal/media/forwarding/publication_test.go index 79ce6287d..2d5c221c2 100644 --- a/internal/media/forwarding/publication_test.go +++ b/internal/media/forwarding/publication_test.go @@ -105,7 +105,6 @@ func TestPublicationSimulcastReuseAudioDemandAndClose(t *testing.T) { check(mediaEngine.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: "urn:ietf:params:rtp-hdrext:sdes:mid"}, webrtc.RTPCodecTypeVideo)) registry := &interceptor.Registry{} check(webrtc.RegisterDefaultInterceptors(mediaEngine, registry)) - check(webrtc.ConfigureTWCCSender(mediaEngine, registry)) receiver, err := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine), webrtc.WithInterceptorRegistry(registry), webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) check(err) defer receiver.Close() diff --git a/internal/media/forwarding/recovery_test.go b/internal/media/forwarding/recovery_test.go index 7102f4e90..b9d764467 100644 --- a/internal/media/forwarding/recovery_test.go +++ b/internal/media/forwarding/recovery_test.go @@ -70,7 +70,6 @@ func TestTransportConstrainedDormantRecovery(t *testing.T) { check(media.RegisterDefaultCodecs()) registry := &interceptor.Registry{} check(webrtc.RegisterDefaultInterceptors(media, registry)) - check(webrtc.ConfigureTWCCSender(media, registry)) receiver, err := webrtc.NewAPI(webrtc.WithMediaEngine(media), webrtc.WithInterceptorRegistry(registry), webrtc.WithSettingEngine(receiveSettings)).NewPeerConnection(webrtc.Configuration{}) check(err) diff --git a/internal/media/forwarding/transport_test.go b/internal/media/forwarding/transport_test.go index 59d7bda08..b65e44fc5 100644 --- a/internal/media/forwarding/transport_test.go +++ b/internal/media/forwarding/transport_test.go @@ -83,7 +83,6 @@ func TestTransportVideoFeedbackAudioAndClose(t *testing.T) { check(mediaEngine.RegisterDefaultCodecs()) registry := &interceptor.Registry{} check(webrtc.RegisterDefaultInterceptors(mediaEngine, registry)) - check(webrtc.ConfigureTWCCSender(mediaEngine, registry)) reports := &receiverReportGate{} registry.Add(reports) receiver, err := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine), webrtc.WithInterceptorRegistry(registry), From 10c86a4d3f83846eb38116a2bedba2f06054771f Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:44:50 +0800 Subject: [PATCH 083/262] fix(media): avoid stale send queues after connection recovery --- docs/research/node-local-media-design.md | 53 +++++++++-- internal/media/forwarding/publication.go | 30 +++--- internal/media/forwarding/publication_test.go | 38 ++++++-- internal/media/forwarding/transport.go | 29 ++---- internal/media/forwarding/transport_test.go | 92 +++++++++++++++++++ 5 files changed, 190 insertions(+), 52 deletions(-) diff --git a/docs/research/node-local-media-design.md b/docs/research/node-local-media-design.md index 03a8d1176..3a1e4cacb 100644 --- a/docs/research/node-local-media-design.md +++ b/docs/research/node-local-media-design.md @@ -387,18 +387,55 @@ Keep the dependency behind a shared media package. For one video per connection, use the public provisional allocation primitives, with exact typed child state read by its parent; do not parse DebugInfo or run a second multi-track allocator. The candidate now uses LiveKit SendSideBWE and Prober through their public APIs. -Pacing reuses the upstream queue with admission capped by the existing packet -cache bound; canceled probe tickets remain charged until actually dequeued. -The pinned Pion pacer's 1.5 headroom applies only to pacing, not codec or BWE -budgets. Adapted probe lifecycle code retains Apache-2.0 attribution. Probe +Packet sending uses the pinned [LiveKit transport's send-side BWE default](https://github.com/livekit/livekit/blob/v1.13.6/pkg/rtc/transport.go), +`NoQueue`: an asynchronous writer without a second bitrate throttle on already +encoded packets. Admission stays capped by the existing packet cache bound; +canceled probe tickets remain charged until actually dequeued. Congestion +estimation, probes and codec/allocation budgets retain their existing owners. +Adapted probe lifecycle code retains Apache-2.0 attribution. Probe promotion requires the upstream validity criterion and non-congestion signal; an empty or inconclusive attempt cannot promote the estimator's initial ceiling. +#### Send Queue Recovery + +A bounded writer-stall control exposes a limitation in the former SFU queue: +after the estimate falls from 500 to 8 kbps, only three of 32 already-encoded +packets drain in two seconds. The upstream unthrottled writer drains the same +input once the writer resumes. A separate real Windows VP8 codec and +Native forwarding control reproduces stale decoded pictures after roughly one +second of ordinary UDP loss; the packet-count cap alone does not prevent this. +The [Native transport experiment](./nat-traversal.md#integration-cost-and-recovery-controls) +owns the decoded-picture and address-change boundary. + +Congestion callbacks could also promote the estimator's unmeasured 100 Mbps +initial capacity while a probe was pending. Congestion now only reduces the +committed budget; a valid completed non-congesting probe owns increases. +Publication allocation and encoder input use that same committed capacity after +audio reservation. This follows the +[upstream distinction between congestion events and completed probes](https://github.com/livekit/livekit/blob/v1.13.6/pkg/sfu/streamallocator/streamallocator.go), +without introducing another estimator. + +Real decode also exposed a packet-format mismatch at the shared writer: +LiveKit 1.13.6 includes padding in its probe payload, while Pion SRTP serializes +another block from `Header.PaddingSize`. An actual library probe therefore +arrived with 255 bytes of apparent video payload plus 255 bytes of padding. +The adapter now removes the already-included block only at the Pion write +boundary; pacing and TWCC retain the intended packet size. Both publication +and single-child forwarding use that boundary. The regression checks decoded +RTP payload emptiness after SRTP delivery, rather than treating any received +packet as valid video. The primary owners are +[LiveKit probe construction](https://github.com/livekit/livekit/blob/v1.13.6/pkg/sfu/downtrack.go) +and [Pion packet serialization](https://github.com/pion/rtp/blob/v1.10.5/packet.go). + The explicit `TestTransportConstrainedDormantRecovery` virtual-network trace uses real -Pion RTP/TWCC through 6 Mbps -> 500 kbps -> 6 Mbps shaping. It measured a minimum -budget about 497 kbps, stopped the upper output, and restored sustained 1.20 Mbps -after release. Recovery occurred about 4.4 seconds after release in the recorded -run; this is a bounded local trace, not a success-rate or game-latency claim. +Pion RTP/TWCC through 6 Mbps -> 500 kbps -> 6 Mbps shaping. Three consecutive runs +stop the upper output under that constraint and restore sustained 1.20 Mbps after +release. The SDK latency-aware queue passes the same three controls but requires +an extra adapter; it is not retained. Earlier flaky comparisons registered TWCC +twice on the receiver, duplicating feedback; those results do not rank the queues. +Product receivers already use the single default interceptor owner. +No reset, retry owner or minimum quality is added. These are bounded local traces, +not a field diagnosis, success-rate or game-latency claim. Ordinary focused checks cover NACK repair, no-RR probe admission, zero-write termination and exact shutdown. Before runtime cutover, complete actual codec, Browser and physical recovery checks rather than inferring them from this trace. diff --git a/internal/media/forwarding/publication.go b/internal/media/forwarding/publication.go index fa1222f5e..4e3a5cb6e 100644 --- a/internal/media/forwarding/publication.go +++ b/internal/media/forwarding/publication.go @@ -67,16 +67,14 @@ func NewPublication(options TransportOptions) (_ *Publication, err error) { })} publication.bandwidth.target.Store(int64(options.InitialBitrate)) publication.pacer = &publicationPacer{headers: make(map[uint32]publicationHeaders), boundedPacer: newBoundedPacer( - pacer.NewLeakyBucket(log, nil, 5*time.Millisecond, options.InitialBitrate), log, options.Source.maxPackets, - ), bandwidth: publication.bandwidth} + pacer.NewNoQueue(log, nil), log, options.Source.maxPackets, + )} publication.pacer.sender = pacer.NewBase(log, publication.bandwidth) - publication.bandwidth.pacer = publication.pacer publication.allocator = streamallocator.NewStreamAllocator(streamallocator.StreamAllocatorParams{ Config: streamallocator.DefaultStreamAllocatorConfig, BWE: publication.bandwidth, Pacer: publication.pacer, Logger: log, }, true, true) publication.allocator.Start() - publication.pacer.SetBitrate(options.InitialBitrate) media := &webrtc.MediaEngine{} codec := options.Source.Codec() if err = media.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil { @@ -529,7 +527,11 @@ type publicationWriter struct { func (writer *publicationWriter) WriteRTP(header *rtp.Header, payload []byte) (int, error) { written, err := writer.TrackLocalWriter.WriteRTP(header, payload) if err == nil && written > 0 { - writer.bytes.Add(uint64(header.MarshalSize() + len(payload))) + size := header.MarshalSize() + len(payload) + if header.Padding { + size += int(header.PaddingSize) + } + writer.bytes.Add(uint64(size)) if writer.frames != nil && writer.primary && header.Marker && len(payload) > 0 { writer.frames.Add(1) } @@ -563,7 +565,6 @@ type publicationBWE struct { target atomic.Int64 audio atomic.Int64 observed atomic.Bool - pacer *publicationPacer probeMu sync.Mutex completedProbe ccutils.ProbeClusterInfo } @@ -578,11 +579,12 @@ func (bandwidth *publicationBWE) SetBWEListener(listener bwe.BWEListener) { bandwidth.SendSideBWE.SetBWEListener(bandwidth) } func (bandwidth *publicationBWE) OnCongestionStateChange(from, to bwe.CongestionState, capacity int64) { - if capacity > 0 { + // As for an individual Transport, only a completed non-congesting probe + // can raise committed capacity. All consumers use that same accepted value. + if to == bwe.CongestionStateCongested && capacity > 0 && capacity < bandwidth.target.Load() { bandwidth.target.Store(capacity) - bandwidth.pacer.SetBitrate(int(capacity)) } - bandwidth.listener.OnCongestionStateChange(from, to, max(capacity-bandwidth.audio.Load(), 0)) + bandwidth.listener.OnCongestionStateChange(from, to, max(bandwidth.target.Load()-bandwidth.audio.Load(), 0)) } func (bandwidth *publicationBWE) ProbeClusterDone(info ccutils.ProbeClusterInfo) { bandwidth.probeMu.Lock() @@ -601,18 +603,15 @@ func (bandwidth *publicationBWE) ProbeClusterFinalize() (ccutils.ProbeSignal, in // StreamAllocator may upgrade on Inconclusive too. Its capacity must // never promote the estimator's initial sentinel after an empty probe. signal = ccutils.ProbeSignalInconclusive - capacity = min(capacity, bandwidth.target.Load()) } else if signal == ccutils.ProbeSignalNotCongesting && capacity > bandwidth.target.Load() { bandwidth.target.Store(capacity) - bandwidth.pacer.SetBitrate(int(capacity)) } } - return signal, max(capacity-bandwidth.audio.Load(), 0), done + return signal, max(bandwidth.target.Load()-bandwidth.audio.Load(), 0), done } type publicationPacer struct { *boundedPacer - bandwidth *publicationBWE headersMu sync.Mutex headers map[uint32]publicationHeaders } @@ -634,13 +633,8 @@ func (queue *publicationPacer) Enqueue(packet *pacer.Packet) { } queue.boundedPacer.Enqueue(packet) } -func (queue *publicationPacer) StartProbeCluster(info ccutils.ProbeClusterInfo) { - queue.boundedPacer.StartProbeCluster(info) - queue.SetBitrate(info.Goal.DesiredBps + int(queue.bandwidth.audio.Load())) -} func (queue *publicationPacer) EndProbeCluster(id ccutils.ProbeClusterId) ccutils.ProbeClusterInfo { info := queue.boundedPacer.EndProbeCluster(id) queue.cancelProbes(id) - queue.SetBitrate(int(queue.bandwidth.target.Load())) return info } diff --git a/internal/media/forwarding/publication_test.go b/internal/media/forwarding/publication_test.go index 2d5c221c2..a63d4cad8 100644 --- a/internal/media/forwarding/publication_test.go +++ b/internal/media/forwarding/publication_test.go @@ -30,19 +30,19 @@ func TestPublicationEmptyProbeCannotIncreaseCapacity(t *testing.T) { })} bandwidth.target.Store(target) bandwidth.audio.Store(audio) - bandwidth.pacer = &publicationPacer{bandwidth: bandwidth, boundedPacer: newBoundedPacer( - pacer.NewLeakyBucket(log, nil, 5*time.Millisecond, int(target)), log, 500, + queue := &publicationPacer{boundedPacer: newBoundedPacer( + pacer.NewNoQueue(log, nil), log, 500, )} - defer bandwidth.pacer.Stop() + defer queue.Stop() info := ccutils.ProbeClusterInfo{Id: 1, CreatedAt: time.Now(), Goal: ccutils.ProbeClusterGoal{ AvailableBandwidthBps: int(target), ExpectedUsageBps: int(target - audio), DesiredBps: int(target * 2), Duration: bandwidth.ProbeDuration(), }} info.Goal.DesiredBytes = int(float64(info.Goal.DesiredBps) * info.Goal.Duration.Seconds() / 8) bandwidth.ProbeClusterStarting(info) - bandwidth.pacer.StartProbeCluster(info) + queue.StartProbeCluster(info) time.Sleep(info.Goal.Duration) - info = bandwidth.pacer.EndProbeCluster(info.Id) + info = queue.EndProbeCluster(info.Id) if info.Result.Bytes() != 0 || sendsidebwe.DefaultSendSideBWEConfig.CongestionDetector.ProbeSignal.IsValid(info) { t.Fatal("fixture did not complete an empty invalid probe") } @@ -66,6 +66,29 @@ func TestPublicationEmptyProbeCannotIncreaseCapacity(t *testing.T) { } } +type publicationCapacityRecorder struct{ capacity int64 } + +func (recorder *publicationCapacityRecorder) OnCongestionStateChange(_, _ bwe.CongestionState, capacity int64) { + recorder.capacity = capacity +} + +func TestPublicationCongestionUsesCommittedCapacity(t *testing.T) { + recorder := &publicationCapacityRecorder{} + bandwidth := &publicationBWE{listener: recorder} + bandwidth.target.Store(1_200_000) + bandwidth.audio.Store(64_000) + bandwidth.OnCongestionStateChange(bwe.CongestionStateNone, bwe.CongestionStateEarlyWarning, 100_000_000) + bandwidth.OnCongestionStateChange(bwe.CongestionStateEarlyWarning, bwe.CongestionStateCongested, 100_000_000) + if bandwidth.target.Load() != 1_200_000 || recorder.capacity != 1_136_000 { + t.Fatalf("publication promoted an unverified capacity: target=%d allocation=%d", bandwidth.target.Load(), recorder.capacity) + } + bandwidth.OnCongestionStateChange(bwe.CongestionStateCongested, bwe.CongestionStateCongested, 500_000) + bandwidth.OnCongestionStateChange(bwe.CongestionStateCongested, bwe.CongestionStateNone, 800_000) + if bandwidth.target.Load() != 500_000 || recorder.capacity != 436_000 { + t.Fatalf("publication disagrees with its committed audio/video budget: target=%d allocation=%d", bandwidth.target.Load(), recorder.capacity) + } +} + func TestPublicationSimulcastReuseAudioDemandAndClose(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) defer cancel() @@ -225,7 +248,10 @@ func TestPublicationSimulcastReuseAudioDemandAndClose(t *testing.T) { if publication.bandwidth.CongestionState() != bwe.CongestionStateNone || !publication.Counters().Limited { t.Fatalf("clear transport feedback hid the committed publication allocation deficit: state=%v counters=%+v low=%v high=%v", publication.bandwidth.CongestionState(), publication.Counters(), publication.tracks[0].IsDeficient(), publication.tracks[1].IsDeficient()) } - publication.bandwidth.OnCongestionStateChange(bwe.CongestionStateCongested, bwe.CongestionStateCongested, 1_000_000) + // Restore this allocation fixture's capacity explicitly. A congestion event + // cannot prove an increase; this test covers RID reuse, not bandwidth probing. + publication.bandwidth.target.Store(1_000_000) + publication.allocator.SetChannelCapacity(936_000) check(publication.SetActiveCount(1)) if publication.RequiredActiveCount() != 1 { t.Fatal("publication did not reduce its direct demand") diff --git a/internal/media/forwarding/transport.go b/internal/media/forwarding/transport.go index 0ab8a5474..3622f1350 100644 --- a/internal/media/forwarding/transport.go +++ b/internal/media/forwarding/transport.go @@ -136,11 +136,10 @@ func NewTransport(options TransportOptions) (_ *Transport, err error) { }) transport.estimator.SetBWEListener(transport) transport.prober = ccutils.NewProber(ccutils.ProberParams{Listener: transport, Logger: log}) - // Use the pinned Pion pacer's 5 ms service cadence, with LiveKit packet - // ownership and a cap matching this connection's retransmission cache. - transport.pacer = newBoundedPacer(pacer.NewLeakyBucket(log, nil, 5*time.Millisecond, options.InitialBitrate), + // Match LiveKit's send-side BWE default: a reduced estimate must not throttle + // already-encoded frames into a stale backlog after the writer recovers. + transport.pacer = newBoundedPacer(pacer.NewNoQueue(log, nil), log, options.Source.maxPackets) - transport.pacer.SetBitrate(options.InitialBitrate) transport.pacer.sender = pacer.NewBase(log, transport.estimator) transport.pacer.SetPacerProbeObserverListener(transport) track, err := sfu.NewDownTrack(sfu.DownTrackParams{ @@ -312,11 +311,7 @@ func (transport *Transport) SetAudioBitrate(bitrate uint32) { } func (transport *Transport) refreshBudgetLocked() { - bitrate := int(transport.target.Load()) - remaining := int64(bitrate) - int64(transport.audioBitrate) - if transport.probe.Id == ccutils.ProbeClusterIdInvalid || transport.probe.Result.EndTime != 0 { - transport.pacer.SetBitrate(bitrate) - } + remaining := transport.target.Load() - int64(transport.audioBitrate) transport.Output.SetBudget(max(remaining, 0)) } @@ -357,9 +352,11 @@ func (transport *Transport) OnTransportCCFeedback(_ *sfu.DownTrack, packet *rtcp } // HandleTWCCFeedback invokes this synchronously under the transport's event -// lock. Store the framework result without reentering that lock. -func (transport *Transport) OnCongestionStateChange(_, _ bwe.CongestionState, capacity int64) { - if capacity > 0 { +// lock. Congestion can reduce committed capacity; only a valid completed probe +// can increase it. Warning/clear transitions can carry the estimator's initial +// ceiling, especially while a probe is pending, rather than a new measurement. +func (transport *Transport) OnCongestionStateChange(_, state bwe.CongestionState, capacity int64) { + if state == bwe.CongestionStateCongested && capacity > 0 && capacity < transport.target.Load() { transport.target.Store(capacity) } } @@ -457,7 +454,6 @@ func (transport *Transport) driveProbeLocked() { transport.probe = transport.prober.AddCluster(ccutils.ProbeClusterModeUniform, goal) transport.estimator.ProbeClusterStarting(transport.probe) transport.pacer.StartProbeCluster(transport.probe) - transport.pacer.SetBitrate(goal.DesiredBps) transport.Output.SetProbeClusterId(transport.probe.Id) } @@ -470,7 +466,6 @@ func (transport *Transport) stopProbeLocked() { transport.estimator.ProbeClusterDone(transport.probe) transport.prober.Reset(transport.probe) transport.pacer.cancelProbes(transport.probe.Id) - transport.pacer.SetBitrate(int(transport.target.Load())) } // AddCluster returns the exact cluster synchronously. Initializing it there @@ -524,12 +519,6 @@ func newBoundedPacer(packetPacer pacer.Pacer, log logger.Logger, limit int) *bou pending: make(map[*pacedPacket]struct{}), limit: limit} } -func (queue *boundedPacer) SetBitrate(bitrate int) { - // Pion interceptor v0.1.47 gcc.LeakyBucketPacer.SetTargetBitrate uses 1.5 - // pacing headroom. Codec/allocation budgets retain the unscaled BWE result. - queue.Pacer.SetBitrate(int(1.5 * float64(bitrate))) -} - func (queue *boundedPacer) Enqueue(packet *pacer.Packet) { queue.mu.Lock() if queue.closed || len(queue.pending) >= queue.limit { diff --git a/internal/media/forwarding/transport_test.go b/internal/media/forwarding/transport_test.go index b65e44fc5..3a77fce4b 100644 --- a/internal/media/forwarding/transport_test.go +++ b/internal/media/forwarding/transport_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/bwe" "github.com/livekit/livekit-server/pkg/sfu/ccutils" "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/protocol/logger" @@ -343,6 +344,97 @@ func (writer *countingPacketWriter) WriteRTP(header *rtp.Header, payload []byte) func (*countingPacketWriter) Write([]byte) (int, error) { panic("unexpected byte write") } +func TestTransportDoesNotPromoteUnverifiedCongestionCapacity(t *testing.T) { + transport := &Transport{} + transport.target.Store(1_200_000) + // A pending probe can leave the framework's initial 100 Mbps estimate in + // state-change callbacks. It is not proof that the connection can send it. + transport.OnCongestionStateChange(bwe.CongestionStateNone, bwe.CongestionStateEarlyWarning, 100_000_000) + transport.OnCongestionStateChange(bwe.CongestionStateEarlyWarning, bwe.CongestionStateCongested, 100_000_000) + if bitrate, _ := transport.TargetBitrate(); bitrate != 1_200_000 { + t.Fatalf("congestion promoted an unverified capacity: %d", bitrate) + } + transport.OnCongestionStateChange(bwe.CongestionStateCongested, bwe.CongestionStateCongested, 500_000) + transport.OnCongestionStateChange(bwe.CongestionStateCongested, bwe.CongestionStateNone, 800_000) + if bitrate, _ := transport.TargetBitrate(); bitrate != 500_000 { + t.Fatalf("clearing congestion replaced the committed reduction without a probe: %d", bitrate) + } +} + +type stalledPacketWriter struct { + ready, release chan struct{} + written chan struct{} + once sync.Once +} + +func (writer *stalledPacketWriter) WriteRTP(header *rtp.Header, payload []byte) (int, error) { + writer.once.Do(func() { close(writer.ready); <-writer.release }) + writer.written <- struct{}{} + return header.MarshalSize() + len(payload), nil +} + +func (*stalledPacketWriter) Write([]byte) (int, error) { panic("unexpected byte write") } + +func TestSendQueuesDrainAfterBandwidthReduction(t *testing.T) { + for _, kind := range []string{"transport", "publication"} { + t.Run(kind, func(t *testing.T) { + codec := webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, ClockRate: 90_000, + }, PayloadType: 96} + source, err := NewSource(SourceOptions{ID: "queue", StreamID: "queue", Codec: codec, + Formats: []LayerFormat{{Width: 320, Height: 180, Bitrate: 500_000}}, MaxPackets: 500}) + if err != nil { + t.Fatal(err) + } + defer source.Close() + options := TransportOptions{Source: source, ConnectionID: "queue", InitialBitrate: 500_000} + var queue *boundedPacer + if kind == "publication" { + publication, err := NewPublication(options) + if err != nil { + t.Fatal(err) + } + defer publication.Close() + queue = publication.pacer.boundedPacer + } else { + transport, err := NewTransport(options) + if err != nil { + t.Fatal(err) + } + defer transport.Close() + queue = transport.pacer + } + writer := &stalledPacketWriter{ready: make(chan struct{}), release: make(chan struct{}), written: make(chan struct{}, 32)} + release := sync.OnceFunc(func() { close(writer.release) }) + defer release() + for index := range 32 { + header := &rtp.Header{Version: 2, SSRC: 1, SequenceNumber: uint16(index)} + queue.Enqueue(&pacer.Packet{Header: header, HeaderSize: header.MarshalSize(), + Payload: make([]byte, 1200), WriteStream: writer}) + } + select { + case <-writer.ready: + case <-time.After(2 * time.Second): + t.Fatal("transport did not reach the stalled writer") + } + // A short outage reduces the bandwidth estimate after these packets were + // encoded. The recovered writer must not spend tens of seconds sending old + // video before it can deliver a new frame. + queue.SetBitrate(8_000) + release() + deadline := time.NewTimer(2 * time.Second) + defer deadline.Stop() + for range 32 { + select { + case <-writer.written: + case <-deadline.C: + t.Fatalf("writable transport retained old video: pending=%d sent=%d", queue.pendingCount(), queue.stats().Packets) + } + } + }) + } +} + func TestBoundedPacerReleasesPacketsAndIsolatesBlockedQueue(t *testing.T) { log := logger.GetDiscardLogger() sender := pacer.NewPassThrough(log, nil) From 5be56b45bff5edf61470db5721c65deb10e99a77 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:55:28 +0800 Subject: [PATCH 084/262] docs(research): bound Native tunnel handoff evidence --- docs/research/nat-traversal.md | 59 ++++++++++++++++++++++------------ docs/todo.md | 9 ++++-- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 805c25e41..a0b117086 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -656,27 +656,44 @@ acceptance. They do not resolve the gateway-only or prolonged-outage controls above. Supported dependencies, room grants and representative network evidence remain prerequisites for product adoption. -The 2026-10-09 remote-socket overlay extends that comparison through the actual -Native encoded source, sender, receiver and ordinary local playback. It leaves -product files untouched and admits one fixture peer flow per engine. Unchanged -ordinary/Tailcat controls pass, and both handoff directions can retain media -after DERP closure. Stronger fresh-content checks expose a limit: two of three -sender-handoff repetitions pause output for roughly two seconds while ICE stays -connected; one remains paused at the fifteen-second observation boundary. -The outgoing frame counter also stops as the sender estimate falls to about -20 kbps. This locates those gaps at forwarding/adaptation, rather than proving -that the underlay lost the connection. - -The fixed 8×8 input deliberately has no live rate-controllable encoder. Declaring -it rate-controlled removes the later forwarding pauses in three repetitions, -but all three still miss the strict first fresh-content window and later deliver -about 34–37 frames per second instead of the requested 50. This is an ablation, -not a valid capture implementation or a proposed product switch. Actual Native -capture already declares its live lowest output controllable and applies codec -budgets; raw forwarding retains the framework's pause policy. Full App acceptance -must include that encoder owner, local Browser decode and recovery, and useful -picture continuity. A preserved ICE session alone does not establish a net -handoff benefit. No production adaptation rule or timeout changed. +The remote-socket overlay extends that comparison through the actual Native +encoded source, sender, receiver and local playback. It leaves product files +untouched and admits one fixture peer flow per engine. Its fixed 8×8 input +exposes forwarding pauses but cannot establish live encoder recovery; declaring +those fixed bytes rate-controlled is only an ablation. + +The live control instead uses the Windows VP8 helper, Native output-plan owner +and final FFmpeg decode. The top quarter of the input changes from red to blue +after DERP closure, so buffered pictures cannot satisfy fresh-content acceptance. +About one second of ordinary UDP loss can leave the former pacer sending old +video from a queue of hundreds of packets. DownTrack frame counts precede actual +paced writes and cannot prove delivery. The shared media assessment owns the +[pacing comparison](./node-local-media-design.md#send-queue-recovery); +it must pass constrained-bandwidth recovery as well as this short-loss control. + +Viewer address changes expose a separate confound: same-machine playback can +select a LAN address that the experiment then removes. Restricting only the +fixture receiver to loopback separates that leg from the remote underlay. +With this control and the upstream default send queue, all six cases pass: +unchanged ordinary/Tailcat paths, both address-change directions, and short +ordinary/Tailcat loss. Each retains decoded fresh pictures in every post-DERP +sample. These passes do not accept the default local-playback path or the +complete App. Verify its Browser bridge +and existing recovery owner before adopting a local-address policy. +No additional NAT-reachability class, physical capture result or product +Magicsock integration follows from these controls. + +A same-machine Windows Browser control rejects forcing the product bridge to +loopback. Chrome 154 decodes from a Native sender bound to `127.0.0.1`, but +Firefox 155 fails that connection; the same Firefox fixture decodes with the +current wildcard bind and a non-loopback selected pair. Mozilla also keeps +[ICE loopback disabled by default](https://searchfox.org/firefox-main/source/modules/libpref/init/all.js). +These controls establish a compatibility limit, not its complete Browser cause +or public-Site permission behavior. Keep the current address policy. The Host +currently ends sharing when its required local bridge fails; the Viewer delegates +that failure to its existing bounded connection-replacement owner. Complete App +handoff acceptance must cover both, without treating the remote tunnel's survival +as proof that local playback survived. Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof diff --git a/docs/todo.md b/docs/todo.md index b8a620a31..e9fb0be38 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -72,9 +72,12 @@ Implementation needs a concrete useful result within the product model. Isolated local-address changes now preserve the same session under the actual Viewer recovery owner. Verify that bounded benefit through the complete App pipeline and physical networks; it does not establish another reachable NAT class. - The actual Native media overlay can retain the transport but its fixed-input - continuity controls expose forwarding pauses. Include the live encoder and - decoded picture continuity; do not treat the rate-control ablation as a fix. + Live-encoder/decoded-picture controls now separate shared forwarding queue + delay from the local playback leg losing its selected LAN address. The shared + send-queue repair passes constrained-link recovery; verify the App's local + Browser bridge during address changes. A loopback-only policy fails the Firefox + control; retain the current address policy and verify both Host and Viewer + lifecycle behavior before claiming complete App recovery. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) and RTCP/MTU boundaries, retained-flow recovery beyond the passing serial From efd9f6a415ca25fd420620561e9963c0718ddea5 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:07:34 +0800 Subject: [PATCH 085/262] test(gates): distinguish profile application from decoding --- scripts/app-native-host-gate.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/app-native-host-gate.ts b/scripts/app-native-host-gate.ts index 0bf8c6053..c6c0f6d12 100644 --- a/scripts/app-native-host-gate.ts +++ b/scripts/app-native-host-gate.ts @@ -1022,12 +1022,13 @@ async function main(): Promise { result.viewerWidth = viewerState.width; result.viewerHeight = viewerState.height; await assertVideoCodec(cdp, viewer, actualCodec); - stage = "native-4k-quality"; + stage = "native-4k-preset"; await clickHostControl(cdp, host, `document.querySelectorAll('button.lr-tile')[2]`); await waitForValue(deadline => evaluate(cdp!, host, `document.querySelector('.lr-slider input[type="range"]')?.value === '60' && document.querySelector('.lr-sharing-panel')?.getAttribute('aria-busy') === 'false'`, deadline), Boolean, 20_000); + stage = "native-4k-controls"; const framesBefore4K = await evaluate(cdp, viewer, `document.querySelector('video')?.getVideoPlaybackQuality().totalVideoFrames ?? 0`, Date.now() + 5_000); await evaluate(cdp, host, @@ -1038,6 +1039,7 @@ async function main(): Promise { const height = document.querySelector('.lr-video-size input[name="height"]'); return fps?.max === '30' && fps.value === '30' && width?.value === '3840' && height?.value === '2160'; })()`, deadline), Boolean, 10_000); + stage = "native-4k-decode"; result.fourKQualityChanged = await waitForValue(deadline => evaluate(cdp!, viewer, `(() => { const video = document.querySelector('video'); return Boolean(video && window.__piikGateFitsProfile(3840, 2160) && From 726a10507d8697c5c8b5e46116724b4324080e32 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:50:37 +0800 Subject: [PATCH 086/262] fix(media): finalize bandwidth probes from acknowledged traffic --- docs/research/browser-screen-audio-quality.md | 9 +- docs/research/node-local-media-design.md | 35 +- docs/todo.md | 4 +- internal/media/forwarding/publication.go | 32 +- internal/media/forwarding/publication_test.go | 80 +- internal/media/forwarding/recovery_test.go | 3 + internal/media/forwarding/transport.go | 5 +- internal/thirdparty/README.md | 23 + internal/thirdparty/livekit/LICENSE | 202 +++ internal/thirdparty/livekit/NOTICE | 13 + .../sendsidebwe/congestion_detector.go | 1164 +++++++++++++++++ .../livekit/sendsidebwe/packet_group.go | 310 +++++ .../livekit/sendsidebwe/packet_info.go | 45 + .../livekit/sendsidebwe/packet_tracker.go | 168 +++ .../livekit/sendsidebwe/probe_packet_group.go | 143 ++ .../livekit/sendsidebwe/send_side_bwe.go | 151 +++ .../livekit/sendsidebwe/traffic_stats.go | 191 +++ .../livekit/sendsidebwe/twcc_feedback.go | 127 ++ 18 files changed, 2662 insertions(+), 43 deletions(-) create mode 100644 internal/thirdparty/livekit/LICENSE create mode 100644 internal/thirdparty/livekit/NOTICE create mode 100644 internal/thirdparty/livekit/sendsidebwe/congestion_detector.go create mode 100644 internal/thirdparty/livekit/sendsidebwe/packet_group.go create mode 100644 internal/thirdparty/livekit/sendsidebwe/packet_info.go create mode 100644 internal/thirdparty/livekit/sendsidebwe/packet_tracker.go create mode 100644 internal/thirdparty/livekit/sendsidebwe/probe_packet_group.go create mode 100644 internal/thirdparty/livekit/sendsidebwe/send_side_bwe.go create mode 100644 internal/thirdparty/livekit/sendsidebwe/traffic_stats.go create mode 100644 internal/thirdparty/livekit/sendsidebwe/twcc_feedback.go diff --git a/docs/research/browser-screen-audio-quality.md b/docs/research/browser-screen-audio-quality.md index 153c7a605..e51a21eab 100644 --- a/docs/research/browser-screen-audio-quality.md +++ b/docs/research/browser-screen-audio-quality.md @@ -105,8 +105,13 @@ retirement also passed; the Browser run measured stereo separation. An ordinary Browser relay forwarded received tracks to two children at 320 kbps, with advancing decoded audio and successful cleanup. A direct Browser control now keeps requested 256/320 kbps below an explicit 192 kbps receive limit, while a -510 kbps receiver permits both rates. These are local runtime checks; impaired -networks, mobile playback and physical listening remain open. +510 kbps receiver permits both rates. A native real-Opus control through +6 Mbps -> 500 kbps -> 6 Mbps shaping retains decoded audio during live +320/96/320 kbps changes. It exposed the shared +[probe measurement repairs](./node-local-media-design.md#send-queue-recovery), +but video allocation after returning to the high audio rate still needs +verification. These are local runtime checks; physical networks, mobile playback +and listening remain open. RFC 7587 gives an omitted receive bitrate a mode-dependent default, not an unrestricted value, and asks receivers to ignore out-of-range preferences. diff --git a/docs/research/node-local-media-design.md b/docs/research/node-local-media-design.md index 3a1e4cacb..4da1093a4 100644 --- a/docs/research/node-local-media-design.md +++ b/docs/research/node-local-media-design.md @@ -386,7 +386,8 @@ fixes and demand hooks beyond a thin adapter; do not maintain both candidates. Keep the dependency behind a shared media package. For one video per connection, use the public provisional allocation primitives, with exact typed child state read by its parent; do not parse DebugInfo or run a second multi-track allocator. -The candidate now uses LiveKit SendSideBWE and Prober through their public APIs. +The candidate uses LiveKit SendSideBWE and Prober through their public APIs, +with the [scoped measurement repair](../../internal/thirdparty/README.md#livekit-send-side-bandwidth-estimation). Packet sending uses the pinned [LiveKit transport's send-side BWE default](https://github.com/livekit/livekit/blob/v1.13.6/pkg/rtc/transport.go), `NoQueue`: an asynchronous writer without a second bitrate throttle on already encoded packets. Admission stays capped by the existing packet cache bound; @@ -415,6 +416,19 @@ audio reservation. This follows the [upstream distinction between congestion events and completed probes](https://github.com/livekit/livekit/blob/v1.13.6/pkg/sfu/streamallocator/streamallocator.go), without introducing another estimator. +Two upstream measurement defects also affect completed probes. The pinned +[finalizer](https://github.com/livekit/livekit/blob/v1.13.6/pkg/sfu/bwe/sendsidebwe/congestion_detector.go) +returns retained capacity, initially 100 Mbps, rather than the current probe's +acknowledged rate; no feedback can also be classified as non-congesting. The +[probe packet group](https://github.com/livekit/livekit/blob/v1.13.6/pkg/sfu/bwe/sendsidebwe/probe_packet_group.go) +includes the transport's preceding idle interval in the new measurement window. +An 800 kbps probe after five idle seconds therefore either promotes 100 Mbps or, +with only the return-value correction, fails to restore the existing budget. +The scoped repair returns only current probe evidence, marks invalid or +unacknowledged probes inconclusive, and starts timing at the first acknowledged +probe packet. Removing either correction fails the real-TWCC regression. These +changes preserve upstream estimation, loss handling and probe scheduling. + Real decode also exposed a packet-format mismatch at the shared writer: LiveKit 1.13.6 includes padding in its probe payload, while Pion SRTP serializes another block from `Header.PaddingSize`. An actual library probe therefore @@ -427,13 +441,18 @@ packet as valid video. The primary owners are [LiveKit probe construction](https://github.com/livekit/livekit/blob/v1.13.6/pkg/sfu/downtrack.go) and [Pion packet serialization](https://github.com/pion/rtp/blob/v1.10.5/packet.go). -The explicit `TestTransportConstrainedDormantRecovery` virtual-network trace uses real -Pion RTP/TWCC through 6 Mbps -> 500 kbps -> 6 Mbps shaping. Three consecutive runs -stop the upper output under that constraint and restore sustained 1.20 Mbps after -release. The SDK latency-aware queue passes the same three controls but requires -an extra adapter; it is not retained. Earlier flaky comparisons registered TWCC -twice on the receiver, duplicating feedback; those results do not rank the queues. -Product receivers already use the single default interceptor owner. +The explicit `TestTransportConstrainedDormantRecovery` virtual-network trace uses +real Pion RTP/TWCC through 6 Mbps -> 500 kbps -> 6 Mbps shaping. The repaired +library restores sustained 1.20 Mbps video after release, with a budget assertion +that rejects the unmeasured initial capacity. Earlier passing recovery traces +could promote 100 Mbps and do not establish correct budgeting. The SDK +latency-aware queue requires an extra adapter and is not retained. Earlier +flaky comparisons also registered TWCC twice on the receiver, duplicating +feedback; those results do not rank the queues. Product receivers already use +the single default interceptor owner. A separate real Opus 320/96/320 kbps +control retains audio decoding under the same impairment; video can still +return to a lower output after the high audio rate resumes. That allocation +boundary remains open rather than being reported as complete recovery. No reset, retry owner or minimum quality is added. These are bounded local traces, not a field diagnosis, success-rate or game-latency claim. Ordinary focused checks cover NACK repair, no-RR probe admission, zero-write diff --git a/docs/todo.md b/docs/todo.md index e9fb0be38..53052b009 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -57,7 +57,9 @@ Implementation needs a concrete useful result within the product model. limits and framework-owned adaptation. The [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment) retain explicit receive-limit enforcement and the current 320 kbps sender - ceiling; physical listening and impaired-network evidence remain open. + ceiling. A real-Opus impaired-network control retains decoding; verify video + allocation after restoring high audio rates. Physical listening, mobile and + field-network acceptance remain open. New codecs need complete encode/decode/forward support and a late-Viewer policy before inclusion; selecting a codec cannot transcode an existing publication. Confirm live diff --git a/internal/media/forwarding/publication.go b/internal/media/forwarding/publication.go index 4e3a5cb6e..5404dec6a 100644 --- a/internal/media/forwarding/publication.go +++ b/internal/media/forwarding/publication.go @@ -12,10 +12,10 @@ import ( "time" "github.com/TNTcraftHIM/Piik/internal/diagnostics" + "github.com/TNTcraftHIM/Piik/internal/thirdparty/livekit/sendsidebwe" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/bwe" - "github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe" "github.com/livekit/livekit-server/pkg/sfu/ccutils" "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" @@ -561,12 +561,10 @@ func (context publicationAudioContext) WriteStream() webrtc.TrackLocalWriter { r // audio is conservative and does not claim that the estimator acknowledged it. type publicationBWE struct { *sendsidebwe.SendSideBWE - listener bwe.BWEListener - target atomic.Int64 - audio atomic.Int64 - observed atomic.Bool - probeMu sync.Mutex - completedProbe ccutils.ProbeClusterInfo + listener bwe.BWEListener + target atomic.Int64 + audio atomic.Int64 + observed atomic.Bool } func (bandwidth *publicationBWE) HandleTWCCFeedback(report *rtcp.TransportLayerCC) { @@ -586,26 +584,10 @@ func (bandwidth *publicationBWE) OnCongestionStateChange(from, to bwe.Congestion } bandwidth.listener.OnCongestionStateChange(from, to, max(bandwidth.target.Load()-bandwidth.audio.Load(), 0)) } -func (bandwidth *publicationBWE) ProbeClusterDone(info ccutils.ProbeClusterInfo) { - bandwidth.probeMu.Lock() - defer bandwidth.probeMu.Unlock() - bandwidth.completedProbe = info - bandwidth.SendSideBWE.ProbeClusterDone(info) -} func (bandwidth *publicationBWE) ProbeClusterFinalize() (ccutils.ProbeSignal, int64, bool) { - bandwidth.probeMu.Lock() - defer bandwidth.probeMu.Unlock() signal, capacity, done := bandwidth.SendSideBWE.ProbeClusterFinalize() - if done { - valid := sendsidebwe.DefaultSendSideBWEConfig.CongestionDetector.ProbeSignal.IsValid(bandwidth.completedProbe) - bandwidth.completedProbe = ccutils.ProbeClusterInfoInvalid - if !valid { - // StreamAllocator may upgrade on Inconclusive too. Its capacity must - // never promote the estimator's initial sentinel after an empty probe. - signal = ccutils.ProbeSignalInconclusive - } else if signal == ccutils.ProbeSignalNotCongesting && capacity > bandwidth.target.Load() { - bandwidth.target.Store(capacity) - } + if done && signal == ccutils.ProbeSignalNotCongesting && capacity > bandwidth.target.Load() { + bandwidth.target.Store(capacity) } return signal, max(bandwidth.target.Load()-bandwidth.audio.Load(), 0), done } diff --git a/internal/media/forwarding/publication_test.go b/internal/media/forwarding/publication_test.go index a63d4cad8..bf5d36cd3 100644 --- a/internal/media/forwarding/publication_test.go +++ b/internal/media/forwarding/publication_test.go @@ -9,13 +9,15 @@ import ( "time" "github.com/TNTcraftHIM/Piik/internal/media/encoded" + "github.com/TNTcraftHIM/Piik/internal/thirdparty/livekit/sendsidebwe" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/bwe" - "github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe" "github.com/livekit/livekit-server/pkg/sfu/ccutils" "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/protocol/logger" "github.com/pion/interceptor" + "github.com/pion/interceptor/pkg/twcc" + "github.com/pion/rtcp" "github.com/pion/rtp/codecs" "github.com/pion/webrtc/v4" "github.com/pion/webrtc/v4/pkg/media" @@ -54,9 +56,6 @@ func TestPublicationEmptyProbeCannotIncreaseCapacity(t *testing.T) { if signal != ccutils.ProbeSignalInconclusive || capacity > target-audio || bandwidth.target.Load() != target { t.Fatalf("empty probe raised publication capacity: signal=%v capacity=%d target=%d", signal, capacity, bandwidth.target.Load()) } - if bandwidth.completedProbe.Id != ccutils.ProbeClusterIdInvalid { - t.Fatal("completed probe was retained after finalization") - } break } if time.Now().After(deadline) { @@ -66,6 +65,79 @@ func TestPublicationEmptyProbeCannotIncreaseCapacity(t *testing.T) { } } +func TestPublicationProbeUsesAcknowledgedCapacity(t *testing.T) { + for _, received := range []bool{true, false} { + t.Run(map[bool]string{true: "received", false: "no-feedback"}[received], func(t *testing.T) { + checkPublicationProbeCapacity(t, received) + }) + } +} + +func checkPublicationProbeCapacity(t *testing.T, received bool) { + t.Helper() + const target = int64(100_000) + const audio = int64(64_000) + bandwidth := &publicationBWE{SendSideBWE: sendsidebwe.NewSendSideBWE(sendsidebwe.SendSideBWEParams{ + Config: sendsidebwe.DefaultSendSideBWEConfig, Logger: logger.GetDiscardLogger(), + })} + bandwidth.target.Store(target) + bandwidth.audio.Store(audio) + bandwidth.SetBWEListener(&publicationCapacityRecorder{}) + // A real non-congesting probe after reset still carries the library's + // initial capacity. A successful 800 kbps probe cannot prove 100 Mbps. + start := time.Now().Add(-time.Second) + recorder := twcc.NewRecorder(42) + // Establish transport history before the idle interval. The initial packet + // has relative time zero, so include a subsequent packet as in a live stream. + for _, age := range []time.Duration{6 * time.Second, 5 * time.Second} { + previous := start.Add(-age).UnixMicro() + sequence := bandwidth.RecordPacketSendAndGetSequenceNumber(previous, 200, false, ccutils.ProbeClusterIdInvalid, false) + recorder.Record(42, sequence, previous+10_000) + } + for _, packet := range recorder.BuildFeedbackPacket() { + bandwidth.HandleTWCCFeedback(packet.(*rtcp.TransportLayerCC)) + } + bandwidth.Reset() + info := ccutils.ProbeClusterInfo{Id: 1, CreatedAt: start, Goal: ccutils.ProbeClusterGoal{ + AvailableBandwidthBps: int(target), DesiredBps: 800_000, + Duration: 200 * time.Millisecond, DesiredBytes: 20_000, + }} + bandwidth.ProbeClusterStarting(info) + for index := range 100 { + at := start.Add(time.Duration(index) * 2 * time.Millisecond).UnixMicro() + sequence := bandwidth.RecordPacketSendAndGetSequenceNumber(at, 200, false, info.Id, true) + recorder.Record(42, sequence, at+10_000) + } + info.Result = ccutils.ProbeClusterResult{StartTime: start.UnixNano(), + EndTime: start.Add(info.Goal.Duration).UnixNano(), PacketsProbe: 100, BytesProbe: 20_000, IsCompleted: true} + bandwidth.ProbeClusterDone(info) + if received { + for _, packet := range recorder.BuildFeedbackPacket() { + bandwidth.HandleTWCCFeedback(packet.(*rtcp.TransportLayerCC)) + } + } + deadline := time.Now().Add(sendsidebwe.DefaultSendSideBWEConfig.CongestionDetector.ProbePacketGroup.SettleWaitMax + time.Second) + for { + signal, capacity, done := bandwidth.ProbeClusterFinalize() + if done { + if received { + // Allow the first-packet/time-window difference in the library's + // acknowledged rate, but never its untouched 100 Mbps initial value. + if signal != ccutils.ProbeSignalNotCongesting || capacity <= target-audio || capacity > 1_000_000-audio { + t.Fatalf("probe promoted unmeasured capacity: signal=%v committed=%d sent=%.0f", signal, capacity+audio, info.Result.Bitrate()) + } + } else if signal != ccutils.ProbeSignalInconclusive || capacity != target-audio { + t.Fatalf("unacknowledged probe changed capacity: signal=%v committed=%d", signal, capacity+audio) + } + return + } + if time.Now().After(deadline) { + t.Fatal("probe did not finalize within the library settle bound") + } + time.Sleep(5 * time.Millisecond) + } +} + type publicationCapacityRecorder struct{ capacity int64 } func (recorder *publicationCapacityRecorder) OnCongestionStateChange(_, _ bwe.CongestionState, capacity int64) { diff --git a/internal/media/forwarding/recovery_test.go b/internal/media/forwarding/recovery_test.go index b9d764467..ce003ac33 100644 --- a/internal/media/forwarding/recovery_test.go +++ b/internal/media/forwarding/recovery_test.go @@ -136,6 +136,9 @@ func TestTransportConstrainedDormantRecovery(t *testing.T) { } link.SetRate(capacity) state := transport.Output.State() + if state.VideoBudget > 6_000_000 { + t.Fatalf("probe promoted capacity beyond this fixture's link: %+v", state) + } _, observed := transport.TargetBitrate() high := !observed || state.Target == 1 || state.Prepare == 1 || !state.Paused && state.Current == 1 if elapsed >= 7*time.Second && elapsed < 12*time.Second { diff --git a/internal/media/forwarding/transport.go b/internal/media/forwarding/transport.go index 3622f1350..32ac1521f 100644 --- a/internal/media/forwarding/transport.go +++ b/internal/media/forwarding/transport.go @@ -11,10 +11,10 @@ import ( "time" "github.com/TNTcraftHIM/Piik/internal/diagnostics" + "github.com/TNTcraftHIM/Piik/internal/thirdparty/livekit/sendsidebwe" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/bwe" - "github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe" "github.com/livekit/livekit-server/pkg/sfu/ccutils" "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" @@ -423,8 +423,7 @@ func (transport *Transport) driveProbeLocked() { } if transport.probe.Result.EndTime != 0 { if signal, capacity, finalized := transport.estimator.ProbeClusterFinalize(); finalized { - valid := sendsidebwe.DefaultSendSideBWEConfig.CongestionDetector.ProbeSignal.IsValid(transport.probe) - if valid && signal == ccutils.ProbeSignalNotCongesting && capacity > transport.target.Load() { + if signal == ccutils.ProbeSignalNotCongesting && capacity > transport.target.Load() { transport.target.Store(capacity) } transport.probe = ccutils.ProbeClusterInfoInvalid diff --git a/internal/thirdparty/README.md b/internal/thirdparty/README.md index 1352b9115..5be024b0a 100644 --- a/internal/thirdparty/README.md +++ b/internal/thirdparty/README.md @@ -24,3 +24,26 @@ pass, replace both local module directives with version pins, remove their sourc directories and the extra dependency test patterns in `scripts/check-go.mjs`. Until then, review updates against the named baselines; do not edit Go's shared module cache or apply build-time patches. + +## LiveKit Send-Side Bandwidth Estimation + +`livekit/sendsidebwe` retains only the eight source files from +[LiveKit v1.13.6's package](https://github.com/livekit/livekit/tree/v1.13.6/pkg/sfu/bwe/sendsidebwe), +with its original `LICENSE` and `NOTICE`. It uses the existing pinned LiveKit +interfaces; the rest of LiveKit remains the normal Go dependency. There is no +whole-module fork or build-time patch. + +Two measurement corrections are marked in the modified files: probe finalization +returns the current acknowledged capacity and rejects invalid/empty feedback; +probe timing begins with its first acknowledged packet, excluding pre-probe idle +time. Piik's shared forwarding package uses this copy for both individual +transports and publications. Its regression checks use real TWCC feedback and +retain the no-feedback and idle-before-probe controls. The +[media research owner](../../docs/research/node-local-media-design.md#send-queue-recovery) +records the runtime evidence and remaining limits. + +The package notice collector still includes LiveKit's upstream license and +notice through the other compiled LiveKit packages. When a maintained upstream +version covers both corrections and the same regressions pass, remove this +directory and restore the upstream imports. Review dependency bumps against the +named baseline; do not evolve a separate congestion-control algorithm here. diff --git a/internal/thirdparty/livekit/LICENSE b/internal/thirdparty/livekit/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/internal/thirdparty/livekit/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/internal/thirdparty/livekit/NOTICE b/internal/thirdparty/livekit/NOTICE new file mode 100644 index 000000000..692adc992 --- /dev/null +++ b/internal/thirdparty/livekit/NOTICE @@ -0,0 +1,13 @@ +Copyright 2023 LiveKit, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/internal/thirdparty/livekit/sendsidebwe/congestion_detector.go b/internal/thirdparty/livekit/sendsidebwe/congestion_detector.go new file mode 100644 index 000000000..32fb6bf68 --- /dev/null +++ b/internal/thirdparty/livekit/sendsidebwe/congestion_detector.go @@ -0,0 +1,1164 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Modified by Piik: finalize probes from valid acknowledged measurements, +// not the detector's retained initial capacity. +package sendsidebwe + +import ( + "fmt" + "sync" + "time" + + "github.com/livekit/livekit-server/pkg/sfu/bwe" + "github.com/livekit/livekit-server/pkg/sfu/ccutils" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils/mono" + "github.com/pion/rtcp" + "go.uber.org/zap/zapcore" +) + +// ------------------------------------------------------------------------------- + +type CongestionSignalConfig struct { + MinNumberOfGroups int `yaml:"min_number_of_groups,omitempty"` + MinDuration time.Duration `yaml:"min_duration,omitempty"` +} + +func (c CongestionSignalConfig) IsTriggered(numGroups int, duration int64) bool { + return numGroups >= c.MinNumberOfGroups && duration >= c.MinDuration.Microseconds() +} + +var ( + defaultQueuingDelayEarlyWarningJQRConfig = CongestionSignalConfig{ + MinNumberOfGroups: 2, + MinDuration: 200 * time.Millisecond, + } + + defaultQueuingDelayEarlyWarningDQRConfig = CongestionSignalConfig{ + MinNumberOfGroups: 3, + MinDuration: 300 * time.Millisecond, + } + + defaultLossEarlyWarningJQRConfig = CongestionSignalConfig{ + MinNumberOfGroups: 3, + MinDuration: 300 * time.Millisecond, + } + + defaultLossEarlyWarningDQRConfig = CongestionSignalConfig{ + MinNumberOfGroups: 4, + MinDuration: 400 * time.Millisecond, + } + + defaultQueuingDelayCongestedJQRConfig = CongestionSignalConfig{ + MinNumberOfGroups: 4, + MinDuration: 400 * time.Millisecond, + } + + defaultQueuingDelayCongestedDQRConfig = CongestionSignalConfig{ + MinNumberOfGroups: 5, + MinDuration: 500 * time.Millisecond, + } + + defaultLossCongestedJQRConfig = CongestionSignalConfig{ + MinNumberOfGroups: 6, + MinDuration: 600 * time.Millisecond, + } + + defaultLossCongestedDQRConfig = CongestionSignalConfig{ + MinNumberOfGroups: 6, + MinDuration: 600 * time.Millisecond, + } +) + +// ------------------------------------------------------------------------------- + +type ProbeSignalConfig struct { + MinBytesRatio float64 `yaml:"min_bytes_ratio,omitempty"` + MinDurationRatio float64 `yaml:"min_duration_ratio,omitempty"` + + JQRMinDelay time.Duration `yaml:"jqr_min_delay,omitempty"` + DQRMaxDelay time.Duration `yaml:"dqr_max_delay,omitempty"` + + WeightedLoss WeightedLossConfig `yaml:"weighted_loss,omitempty"` + JQRMinWeightedLoss float64 `yaml:"jqr_min_weighted_loss,omitempty"` + DQRMaxWeightedLoss float64 `yaml:"dqr_max_weighted_loss,omitempty"` +} + +func (p ProbeSignalConfig) IsValid(pci ccutils.ProbeClusterInfo) bool { + return pci.Result.Bytes() > int(p.MinBytesRatio*float64(pci.Goal.DesiredBytes)) && pci.Result.Duration() > time.Duration(p.MinDurationRatio*float64(pci.Goal.Duration)) +} + +func (p ProbeSignalConfig) ProbeSignal(ppg *probePacketGroup) (ccutils.ProbeSignal, int64) { + ts := newTrafficStats(trafficStatsParams{ + Config: p.WeightedLoss, + }) + ts.Merge(ppg.Traffic()) + + pqd := ppg.PropagatedQueuingDelay() + if pqd > p.JQRMinDelay.Microseconds() || ts.WeightedLoss() > p.JQRMinWeightedLoss { + return ccutils.ProbeSignalCongesting, ts.AcknowledgedBitrate() + } + + if pqd < p.DQRMaxDelay.Microseconds() && ts.WeightedLoss() < p.DQRMaxWeightedLoss { + return ccutils.ProbeSignalNotCongesting, ts.AcknowledgedBitrate() + } + + return ccutils.ProbeSignalInconclusive, ts.AcknowledgedBitrate() +} + +var ( + defaultProbeSignalConfig = ProbeSignalConfig{ + MinBytesRatio: 0.5, + MinDurationRatio: 0.5, + + JQRMinDelay: 50 * time.Millisecond, + DQRMaxDelay: 20 * time.Millisecond, + + WeightedLoss: defaultWeightedLossConfig, + JQRMinWeightedLoss: 0.25, + DQRMaxWeightedLoss: 0.1, + } +) + +// ------------------------------------------------------------------------------- + +type queuingRegion int + +const ( + queuingRegionDQR queuingRegion = iota + queuingRegionIndeterminate + queuingRegionJQR +) + +func (q queuingRegion) String() string { + switch q { + case queuingRegionDQR: + return "DQR" + case queuingRegionIndeterminate: + return "INDETERMINATE" + case queuingRegionJQR: + return "JQR" + default: + return fmt.Sprintf("%d", int(q)) + } +} + +// ------------------------------------------------------------------------------- + +type congestionReason int + +const ( + congestionReasonNone congestionReason = iota + congestionReasonQueuingDelay + congestionReasonLoss +) + +func (c congestionReason) String() string { + switch c { + case congestionReasonNone: + return "NONE" + case congestionReasonQueuingDelay: + return "QUEUING_DELAY" + case congestionReasonLoss: + return "LOSS" + default: + return fmt.Sprintf("%d", int(c)) + } +} + +// ------------------------------------------------------------------------------- + +type qdMeasurement struct { + jqrConfig CongestionSignalConfig + dqrConfig CongestionSignalConfig + jqrMinDelay int64 + jqrMinTrendCoefficient float64 + dqrMaxDelay int64 + + numGroups int + numJQRGroups int + numDQRGroups int + minSendTime int64 + maxSendTime int64 + propagatedQueuingDelays []int64 + + isSealed bool + minGroupIdx int + maxGroupIdx int + + queuingRegion queuingRegion +} + +func newQDMeasurement(jqrConfig CongestionSignalConfig, dqrConfig CongestionSignalConfig, jqrMinDelay int64, jqrMinTrendCoefficient float64, dqrMaxDelay int64) *qdMeasurement { + return &qdMeasurement{ + jqrConfig: jqrConfig, + dqrConfig: dqrConfig, + jqrMinDelay: jqrMinDelay, + jqrMinTrendCoefficient: jqrMinTrendCoefficient, + dqrMaxDelay: dqrMaxDelay, + queuingRegion: queuingRegionIndeterminate, + } +} + +func (q *qdMeasurement) ProcessPacketGroup(pg *packetGroup, groupIdx int) { + if q.isSealed { + return + } + + pqd, pqdOk := pg.FinalizedPropagatedQueuingDelay() + if !pqdOk { + return + } + + q.numGroups++ + if q.minGroupIdx == 0 || q.minGroupIdx > groupIdx { + q.minGroupIdx = groupIdx + } + q.maxGroupIdx = max(q.maxGroupIdx, groupIdx) + + minSendTime, maxSendTime := pg.SendWindow() + if q.minSendTime == 0 || minSendTime < q.minSendTime { + q.minSendTime = minSendTime + } + q.maxSendTime = max(q.maxSendTime, maxSendTime) + + q.propagatedQueuingDelays = append(q.propagatedQueuingDelays, pqd) + + switch { + case pqd < q.dqrMaxDelay: + q.numDQRGroups++ + if q.numJQRGroups > 0 { + // broken continuity, seal + q.isSealed = true + } else if q.dqrConfig.IsTriggered(q.numDQRGroups, q.maxSendTime-q.minSendTime) { + q.isSealed = true + q.queuingRegion = queuingRegionDQR + } + + case pqd > q.jqrMinDelay: + q.numJQRGroups++ + if q.numDQRGroups > 0 { + // broken continuity, seal + q.isSealed = true + } else if q.jqrConfig.IsTriggered(q.numJQRGroups, q.maxSendTime-q.minSendTime) && q.trendCoefficient() > q.jqrMinTrendCoefficient { + q.isSealed = true + q.queuingRegion = queuingRegionJQR + } + + default: + if q.numDQRGroups > 0 || q.numJQRGroups > 0 { + // broken continuity, seal + q.isSealed = true + } + } +} + +func (q *qdMeasurement) IsSealed() bool { + return q.isSealed +} + +func (q *qdMeasurement) QueuingRegion() queuingRegion { + return q.queuingRegion +} + +func (q *qdMeasurement) GroupRange() (int, int) { + return max(0, q.minGroupIdx), max(0, q.maxGroupIdx) +} + +func (q *qdMeasurement) MarshalLogObject(e zapcore.ObjectEncoder) error { + if q == nil { + return nil + } + + e.AddInt("numGroups", q.numGroups) + e.AddInt("numJQRGroups", q.numJQRGroups) + e.AddInt("numDQRGroups", q.numDQRGroups) + e.AddInt64("minSendTime", q.minSendTime) + e.AddInt64("maxSendTime", q.maxSendTime) + e.AddArray("propagatedQueuingDelays", logger.Int64Slice(q.propagatedQueuingDelays)) + e.AddFloat64("trendCoefficient", q.trendCoefficient()) + e.AddDuration("duration", time.Duration((q.maxSendTime-q.minSendTime)*1000)) + e.AddBool("isSealed", q.isSealed) + e.AddInt("minGroupIdx", q.minGroupIdx) + e.AddInt("maxGroupIdx", q.maxGroupIdx) + e.AddString("queuingRegion", q.queuingRegion.String()) + return nil +} + +func (q *qdMeasurement) trendCoefficient() float64 { + concordantPairs := 0 + discordantPairs := 0 + + // the packet groups are processed from newest to oldest, + // so a concordant pair is when the value drops, + // i. e. the propagated queuing delay is increasing if newer (earlier entity in slice) is higher than older (later entity in slice) + for i := 0; i < len(q.propagatedQueuingDelays)-1; i++ { + for j := i + 1; j < len(q.propagatedQueuingDelays); j++ { + if q.propagatedQueuingDelays[i] > q.propagatedQueuingDelays[j] { + concordantPairs++ + } else if q.propagatedQueuingDelays[i] < q.propagatedQueuingDelays[j] { + discordantPairs++ + } + } + } + + if (concordantPairs + discordantPairs) == 0 { + // if the min requirements is only one sample, trend calculation is not possible, declare highest trend value + if len(q.propagatedQueuingDelays) == 1 { + return 1.0 + } + + return 0.0 + } + + return (float64(concordantPairs) - float64(discordantPairs)) / (float64(concordantPairs) + float64(discordantPairs)) +} + +// ------------------------------------------------------------------------------- + +type lossMeasurement struct { + jqrConfig CongestionSignalConfig + dqrConfig CongestionSignalConfig + jqrMinLoss float64 + dqrMaxLoss float64 + + numGroups int + ts *trafficStats + + isJQRSealed bool + isDQRSealed bool + minGroupIdx int + maxGroupIdx int + + weightedLoss float64 + + queuingRegion queuingRegion +} + +func newLossMeasurement( + jqrConfig CongestionSignalConfig, + dqrConfig CongestionSignalConfig, + weightedLossConfig WeightedLossConfig, + jqrMinLoss float64, + dqrMaxLoss float64, + logger logger.Logger, +) *lossMeasurement { + return &lossMeasurement{ + jqrConfig: jqrConfig, + dqrConfig: dqrConfig, + jqrMinLoss: jqrMinLoss, + dqrMaxLoss: dqrMaxLoss, + ts: newTrafficStats(trafficStatsParams{ + Config: weightedLossConfig, + Logger: logger, + }), + queuingRegion: queuingRegionIndeterminate, + } +} + +func (l *lossMeasurement) ProcessPacketGroup(pg *packetGroup, groupIdx int) { + if (l.isJQRSealed && l.isDQRSealed) || !pg.IsFinalized() { + return + } + + l.numGroups++ + if l.minGroupIdx == 0 || l.minGroupIdx > groupIdx { + l.minGroupIdx = groupIdx + } + l.maxGroupIdx = max(l.maxGroupIdx, groupIdx) + + l.ts.Merge(pg.Traffic()) + + if !l.isJQRSealed && l.jqrConfig.IsTriggered(l.numGroups, l.ts.Duration()) { + l.isJQRSealed = true + + weightedLoss := l.ts.WeightedLoss() + if weightedLoss > l.jqrMinLoss { + l.weightedLoss = weightedLoss + l.queuingRegion = queuingRegionJQR + l.isDQRSealed = true // seal DQR also as queuing region has been determined + return + } + } + + if !l.isDQRSealed && l.dqrConfig.IsTriggered(l.numGroups, l.ts.Duration()) { + l.isDQRSealed = true + + weightedLoss := l.ts.WeightedLoss() + if weightedLoss < l.dqrMaxLoss { + l.weightedLoss = weightedLoss + l.queuingRegion = queuingRegionDQR + l.isJQRSealed = true // seal JQR also as queuing region has been determined + return + } + } +} + +func (l *lossMeasurement) IsSealed() bool { + return l.isJQRSealed && l.isDQRSealed +} + +func (l *lossMeasurement) QueuingRegion() queuingRegion { + return l.queuingRegion +} + +func (l *lossMeasurement) GroupRange() (int, int) { + return max(0, l.minGroupIdx), max(0, l.maxGroupIdx) +} + +func (l *lossMeasurement) MarshalLogObject(e zapcore.ObjectEncoder) error { + if l == nil { + return nil + } + + e.AddInt("numGroups", l.numGroups) + e.AddObject("ts", l.ts) + e.AddBool("isJQRSealed", l.isJQRSealed) + e.AddBool("isDQRSealed", l.isDQRSealed) + e.AddInt("minGroupIdx", l.minGroupIdx) + e.AddInt("maxGroupIdx", l.maxGroupIdx) + e.AddFloat64("weightedLoss", l.weightedLoss) + e.AddString("queuingRegion", l.queuingRegion.String()) + return nil +} + +// ------------------------------------------------------------------------------- + +type CongestionDetectorConfig struct { + PacketGroup PacketGroupConfig `yaml:"packet_group,omitempty"` + PacketGroupMaxAge time.Duration `yaml:"packet_group_max_age,omitempty"` + + ProbePacketGroup ProbePacketGroupConfig `yaml:"probe_packet_group,omitempty"` + ProbeRegulator ccutils.ProbeRegulatorConfig `yaml:"probe_regulator,omitempty"` + ProbeSignal ProbeSignalConfig `yaml:"probe_signal,omitempty"` + + JQRMinDelay time.Duration `yaml:"jqr_min_delay,omitempty"` + JQRMinTrendCoefficient float64 `yaml:"jqr_min_trend_coefficient,omitempty"` + DQRMaxDelay time.Duration `yaml:"dqr_max_delay,omitempty"` + + WeightedLoss WeightedLossConfig `yaml:"weighted_loss,omitempty"` + JQRMinWeightedLoss float64 `yaml:"jqr_min_weighted_loss,omitempty"` + DQRMaxWeightedLoss float64 `yaml:"dqr_max_weighted_loss,omitempty"` + + QueuingDelayEarlyWarningJQR CongestionSignalConfig `yaml:"queuing_delay_early_warning_jqr,omitempty"` + QueuingDelayEarlyWarningDQR CongestionSignalConfig `yaml:"queuing_delay_early_warning_dqr,omitempty"` + LossEarlyWarningJQR CongestionSignalConfig `yaml:"loss_early_warning_jqr,omitempty"` + LossEarlyWarningDQR CongestionSignalConfig `yaml:"loss_early_warning_dqr,omitempty"` + + QueuingDelayCongestedJQR CongestionSignalConfig `yaml:"queuing_delay_congested_jqr,omitempty"` + QueuingDelayCongestedDQR CongestionSignalConfig `yaml:"queuing_delay_congested_dqr,omitempty"` + LossCongestedJQR CongestionSignalConfig `yaml:"loss_congested_jqr,omitempty"` + LossCongestedDQR CongestionSignalConfig `yaml:"loss_congested_dqr,omitempty"` + + CongestedCTRTrend ccutils.TrendDetectorConfig `yaml:"congested_ctr_trend,omitempty"` + CongestedCTREpsilon float64 `yaml:"congested_ctr_epsilon,omitempty"` + CongestedPacketGroup PacketGroupConfig `yaml:"congested_packet_group,omitempty"` + + EstimationWindowDuration time.Duration `yaml:"estimaton_window_duration,omitempty"` +} + +var ( + defaultTrendDetectorConfigCongestedCTR = ccutils.TrendDetectorConfig{ + RequiredSamples: 4, + RequiredSamplesMin: 2, + DownwardTrendThreshold: -0.5, + DownwardTrendMaxWait: 2 * time.Second, + CollapseThreshold: 500 * time.Millisecond, + ValidityWindow: 10 * time.Second, + } + + defaultCongestedPacketGroupConfig = PacketGroupConfig{ + MinPackets: 20, + MaxWindowDuration: 150 * time.Millisecond, + } + + defaultCongestionDetectorConfig = CongestionDetectorConfig{ + PacketGroup: defaultPacketGroupConfig, + PacketGroupMaxAge: 10 * time.Second, + + ProbePacketGroup: defaultProbePacketGroupConfig, + ProbeRegulator: ccutils.DefaultProbeRegulatorConfig, + ProbeSignal: defaultProbeSignalConfig, + + JQRMinDelay: 50 * time.Millisecond, + JQRMinTrendCoefficient: 0.8, + DQRMaxDelay: 20 * time.Millisecond, + + WeightedLoss: defaultWeightedLossConfig, + JQRMinWeightedLoss: 0.25, + DQRMaxWeightedLoss: 0.1, + + QueuingDelayEarlyWarningJQR: defaultQueuingDelayEarlyWarningJQRConfig, + QueuingDelayEarlyWarningDQR: defaultQueuingDelayEarlyWarningDQRConfig, + LossEarlyWarningJQR: defaultLossEarlyWarningJQRConfig, + LossEarlyWarningDQR: defaultLossEarlyWarningDQRConfig, + + QueuingDelayCongestedJQR: defaultQueuingDelayCongestedJQRConfig, + QueuingDelayCongestedDQR: defaultQueuingDelayCongestedDQRConfig, + LossCongestedJQR: defaultLossCongestedJQRConfig, + LossCongestedDQR: defaultLossCongestedDQRConfig, + + CongestedCTRTrend: defaultTrendDetectorConfigCongestedCTR, + CongestedCTREpsilon: 0.05, + CongestedPacketGroup: defaultCongestedPacketGroupConfig, + + EstimationWindowDuration: time.Second, + } +) + +// ------------------------------------------------------------------------------- + +type congestionDetectorParams struct { + Config CongestionDetectorConfig + Logger logger.Logger +} + +type congestionDetector struct { + params congestionDetectorParams + + lock sync.Mutex + + rtt float64 + + *packetTracker + twccFeedback *twccFeedback + + packetGroups []*packetGroup + + probePacketGroup *probePacketGroup + probeRegulator *ccutils.ProbeRegulator + + estimatedAvailableChannelCapacity int64 + estimateTrafficStats *trafficStats + + congestionState bwe.CongestionState + congestionStateSwitchedAt time.Time + + congestedCTRTrend *ccutils.TrendDetector[float64] + congestedTrafficStats *trafficStats + congestedPacketGroup *packetGroup + + congestionReason congestionReason + qdMeasurement *qdMeasurement + lossMeasurement *lossMeasurement + + bweListener bwe.BWEListener +} + +func newCongestionDetector(params congestionDetectorParams) *congestionDetector { + c := &congestionDetector{ + params: params, + packetTracker: newPacketTracker(packetTrackerParams{Logger: params.Logger}), + twccFeedback: newTWCCFeedback(twccFeedbackParams{Logger: params.Logger}), + } + c.Reset() + + return c +} + +func (c *congestionDetector) Reset() { + c.lock.Lock() + defer c.lock.Unlock() + + c.rtt = bwe.DefaultRTT + + c.packetGroups = nil + + c.probePacketGroup = nil + c.probeRegulator = ccutils.NewProbeRegulator(ccutils.ProbeRegulatorParams{ + Config: c.params.Config.ProbeRegulator, + Logger: c.params.Logger, + }) + + c.estimatedAvailableChannelCapacity = 100_000_000 + c.estimateTrafficStats = nil + + c.congestionState = bwe.CongestionStateNone + c.congestionStateSwitchedAt = mono.Now() + + c.clearCTRTrend() + + c.congestionReason = congestionReasonNone + c.qdMeasurement = nil + c.lossMeasurement = nil +} + +func (c *congestionDetector) SetBWEListener(bweListener bwe.BWEListener) { + c.lock.Lock() + defer c.lock.Unlock() + + c.bweListener = bweListener +} + +func (c *congestionDetector) getBWEListener() bwe.BWEListener { + c.lock.Lock() + defer c.lock.Unlock() + + return c.bweListener +} + +func (c *congestionDetector) HandleTWCCFeedback(report *rtcp.TransportLayerCC) { + c.lock.Lock() + recvRefTime, _ := c.twccFeedback.ProcessReport(report, mono.Now()) + + if len(c.packetGroups) == 0 { + c.packetGroups = append( + c.packetGroups, + newPacketGroup( + packetGroupParams{ + Config: c.params.Config.PacketGroup, + WeightedLoss: c.params.Config.WeightedLoss, + Logger: c.params.Logger, + }, + 0, + ), + ) + } + + pg := c.packetGroups[len(c.packetGroups)-1] + trackPacketGroup := func(pi *packetInfo, sendDelta, recvDelta int64, isLost bool) { + if pi == nil { + return + } + + c.updateCTRTrend(pi, sendDelta, recvDelta, isLost) + + if c.probePacketGroup != nil { + c.probePacketGroup.Add(pi, sendDelta, recvDelta, isLost) + } + + err := pg.Add(pi, sendDelta, recvDelta, isLost) + if err == nil { + return + } + + if err == errGroupFinalized { + // previous group ended, start a new group + pg = newPacketGroup( + packetGroupParams{ + Config: c.params.Config.PacketGroup, + WeightedLoss: c.params.Config.WeightedLoss, + Logger: c.params.Logger, + }, + pg.PropagatedQueuingDelay(), + ) + c.packetGroups = append(c.packetGroups, pg) + + if err = pg.Add(pi, sendDelta, recvDelta, isLost); err != nil { + c.params.Logger.Warnw("send side bwe: could not add packet to new packet group", err, "packetInfo", pi, "packetGroup", pg) + } + return + } + + // try an older group + for idx := len(c.packetGroups) - 2; idx >= 0; idx-- { + opg := c.packetGroups[idx] + if err := opg.Add(pi, sendDelta, recvDelta, isLost); err == nil { + return + } else if err == errGroupFinalized { + c.params.Logger.Infow("send side bwe: unexpected finalized group", "packetInfo", pi, "packetGroup", opg) + } + } + } + + // 1. go through the TWCC feedback report and record receive time as reported by remote + // 2. process acknowledged packet and group them + // + // losses are not recorded if a feedback report is completely lost. + // RFC recommends treating lost reports by ignoring packets that would have been in it. + // ----------------------------------------------------------------------------------- + // | From a congestion control perspective, lost feedback messages are | + // | handled by ignoring packets which would have been reported as lost or | + // | received in the lost feedback messages. This behavior is similar to | + // | how a lost RTCP receiver report is handled. | + // ----------------------------------------------------------------------------------- + // Reference: https://datatracker.ietf.org/doc/html/draft-holmer-rmcat-transport-wide-cc-extensions-01#page-4 + sequenceNumber := report.BaseSequenceNumber + endSequenceNumberExclusive := sequenceNumber + report.PacketStatusCount + deltaIdx := 0 + processSymbol := func(symbol uint16) { + recvTime := int64(0) + isLost := false + if symbol != rtcp.TypeTCCPacketNotReceived { + recvRefTime += report.RecvDeltas[deltaIdx].Delta + deltaIdx++ + + recvTime = recvRefTime + } else { + isLost = true + } + pi, sendDelta, recvDelta := c.packetTracker.RecordPacketIndicationFromRemote(sequenceNumber, recvTime) + if pi.sendTime != 0 { + trackPacketGroup(&pi, sendDelta, recvDelta, isLost) + } + sequenceNumber++ + } + for _, chunk := range report.PacketChunks { + if sequenceNumber == endSequenceNumberExclusive { + break + } + + switch chunk := chunk.(type) { + case *rtcp.RunLengthChunk: + for i := uint16(0); i < chunk.RunLength; i++ { + if sequenceNumber == endSequenceNumberExclusive { + break + } + + processSymbol(chunk.PacketStatusSymbol) + } + + case *rtcp.StatusVectorChunk: + for _, symbol := range chunk.SymbolList { + if sequenceNumber == endSequenceNumberExclusive { + break + } + + processSymbol(symbol) + } + } + } + + c.prunePacketGroups() + shouldNotify, fromState, toState, committedChannelCapacity := c.congestionDetectionStateMachine() + c.lock.Unlock() + + if shouldNotify { + if bweListener := c.getBWEListener(); bweListener != nil { + bweListener.OnCongestionStateChange(fromState, toState, committedChannelCapacity) + } + } +} + +func (c *congestionDetector) UpdateRTT(rtt float64) { + c.lock.Lock() + defer c.lock.Unlock() + + if rtt == 0 { + c.rtt = bwe.DefaultRTT + } else { + if c.rtt == 0 { + c.rtt = rtt + } else { + c.rtt = bwe.RTTSmoothingFactor*rtt + (1.0-bwe.RTTSmoothingFactor)*c.rtt + } + } +} + +func (c *congestionDetector) CongestionState() bwe.CongestionState { + c.lock.Lock() + defer c.lock.Unlock() + + return c.congestionState +} + +func (c *congestionDetector) CanProbe() bool { + c.lock.Lock() + defer c.lock.Unlock() + + return c.congestionState == bwe.CongestionStateNone && c.probePacketGroup == nil && c.probeRegulator.CanProbe() +} + +func (c *congestionDetector) ProbeDuration() time.Duration { + c.lock.Lock() + defer c.lock.Unlock() + + return c.probeRegulator.ProbeDuration() +} + +func (c *congestionDetector) ProbeClusterStarting(pci ccutils.ProbeClusterInfo) { + c.lock.Lock() + defer c.lock.Unlock() + + c.probePacketGroup = newProbePacketGroup( + probePacketGroupParams{ + Config: c.params.Config.ProbePacketGroup, + WeightedLoss: c.params.Config.WeightedLoss, + Logger: c.params.Logger, + }, + pci, + ) + + c.packetTracker.ProbeClusterStarting(pci.Id) +} + +func (c *congestionDetector) ProbeClusterDone(pci ccutils.ProbeClusterInfo) { + c.lock.Lock() + defer c.lock.Unlock() + + c.packetTracker.ProbeClusterDone(pci.Id) + if c.probePacketGroup != nil { + c.probePacketGroup.ProbeClusterDone(pci) + } +} + +func (c *congestionDetector) ProbeClusterIsGoalReached() bool { + c.lock.Lock() + defer c.lock.Unlock() + + if c.probePacketGroup == nil || c.congestionState != bwe.CongestionStateNone { + return false + } + + pci := c.probePacketGroup.ProbeClusterInfo() + if !c.params.Config.ProbeSignal.IsValid(pci) { + return false + } + + probeSignal, estimatedAvailableChannelCapacity := c.params.Config.ProbeSignal.ProbeSignal(c.probePacketGroup) + return probeSignal != ccutils.ProbeSignalNotCongesting && estimatedAvailableChannelCapacity > int64(pci.Goal.DesiredBps) +} + +func (c *congestionDetector) ProbeClusterFinalize() (ccutils.ProbeSignal, int64, bool) { + c.lock.Lock() + defer c.lock.Unlock() + + if c.probePacketGroup == nil { + return ccutils.ProbeSignalInconclusive, 0, false + } + + pci, isFinalized := c.probePacketGroup.MaybeFinalizeProbe(c.packetTracker.ProbeMaxSequenceNumber(), c.rtt) + if !isFinalized { + return ccutils.ProbeSignalInconclusive, 0, isFinalized + } + + isSignalValid := c.params.Config.ProbeSignal.IsValid(pci) + c.params.Logger.Infow( + "send side bwe: probe finalized", + "isSignalValid", isSignalValid, + "probeClusterInfo", pci, + "probePacketGroup", c.probePacketGroup, + "congestionState", c.congestionState, + "rtt", c.rtt, + ) + + // if congestion signal changed during probe, defer to that signal + if c.congestionState != bwe.CongestionStateNone { + probeSignal := ccutils.ProbeSignalCongesting + c.probeRegulator.ProbeSignal(probeSignal, pci.CreatedAt) + c.probePacketGroup = nil + return probeSignal, c.estimatedAvailableChannelCapacity, true + } + + probeSignal, estimatedAvailableChannelCapacity := c.params.Config.ProbeSignal.ProbeSignal(c.probePacketGroup) + if !isSignalValid || estimatedAvailableChannelCapacity <= 0 { + probeSignal = ccutils.ProbeSignalInconclusive + } + if probeSignal == ccutils.ProbeSignalNotCongesting && estimatedAvailableChannelCapacity > c.estimatedAvailableChannelCapacity { + c.estimatedAvailableChannelCapacity = estimatedAvailableChannelCapacity + } + + c.probeRegulator.ProbeSignal(probeSignal, pci.CreatedAt) + c.probePacketGroup = nil + return probeSignal, estimatedAvailableChannelCapacity, true +} + +func (c *congestionDetector) prunePacketGroups() { + if len(c.packetGroups) == 0 { + return + } + + threshold, ok := c.packetTracker.BaseSendTimeThreshold(c.params.Config.PacketGroupMaxAge.Microseconds()) + if !ok { + return + } + + for idx, pg := range c.packetGroups { + if mst := pg.MinSendTime(); mst > threshold { + c.packetGroups = c.packetGroups[idx:] + return + } + } +} + +func (c *congestionDetector) updateCongestionSignal( + qdJQRConfig CongestionSignalConfig, + qdDQRConfig CongestionSignalConfig, + lossJQRConfig CongestionSignalConfig, + lossDQRConfig CongestionSignalConfig, +) queuingRegion { + c.qdMeasurement = newQDMeasurement( + qdJQRConfig, + qdDQRConfig, + c.params.Config.JQRMinDelay.Microseconds(), + c.params.Config.JQRMinTrendCoefficient, + c.params.Config.DQRMaxDelay.Microseconds(), + ) + c.lossMeasurement = newLossMeasurement( + lossJQRConfig, + lossDQRConfig, + c.params.Config.WeightedLoss, + c.params.Config.JQRMinWeightedLoss, + c.params.Config.DQRMaxWeightedLoss, + c.params.Logger, + ) + + var idx int + for idx = len(c.packetGroups) - 1; idx >= 0; idx-- { + pg := c.packetGroups[idx] + c.qdMeasurement.ProcessPacketGroup(pg, idx) + c.lossMeasurement.ProcessPacketGroup(pg, idx) + + // if both measurements have enough data to make a decision, stop processing groups + if c.qdMeasurement.IsSealed() && c.lossMeasurement.IsSealed() { + break + } + } + + qr := queuingRegionIndeterminate + qdQueuingRegion := c.qdMeasurement.QueuingRegion() + lossQueuingRegion := c.lossMeasurement.QueuingRegion() + switch { + case qdQueuingRegion == queuingRegionJQR: + qr = queuingRegionJQR + c.congestionReason = congestionReasonQueuingDelay + case lossQueuingRegion == queuingRegionJQR: + qr = queuingRegionJQR + c.congestionReason = congestionReasonLoss + case qdQueuingRegion == queuingRegionDQR && lossQueuingRegion == queuingRegionDQR: + qr = queuingRegionDQR + c.congestionReason = congestionReasonNone + } + + return qr +} + +func (c *congestionDetector) updateEarlyWarningSignal() queuingRegion { + return c.updateCongestionSignal( + c.params.Config.QueuingDelayEarlyWarningJQR, + c.params.Config.QueuingDelayEarlyWarningDQR, + c.params.Config.LossEarlyWarningJQR, + c.params.Config.LossEarlyWarningDQR, + ) +} + +func (c *congestionDetector) updateCongestedSignal() queuingRegion { + return c.updateCongestionSignal( + c.params.Config.QueuingDelayCongestedJQR, + c.params.Config.QueuingDelayCongestedDQR, + c.params.Config.LossCongestedJQR, + c.params.Config.LossCongestedDQR, + ) +} + +func (c *congestionDetector) congestionDetectionStateMachine() (bool, bwe.CongestionState, bwe.CongestionState, int64) { + fromState := c.congestionState + toState := c.congestionState + + switch fromState { + case bwe.CongestionStateNone: + if c.updateEarlyWarningSignal() == queuingRegionJQR { + toState = bwe.CongestionStateEarlyWarning + } + + case bwe.CongestionStateEarlyWarning: + if c.updateCongestedSignal() == queuingRegionJQR { + toState = bwe.CongestionStateCongested + } else if c.updateEarlyWarningSignal() == queuingRegionDQR { + toState = bwe.CongestionStateNone + } + + case bwe.CongestionStateCongested: + if c.updateCongestedSignal() == queuingRegionDQR { + toState = bwe.CongestionStateNone + } + } + + shouldNotify := false + if toState != fromState { + c.estimateAvailableChannelCapacity() + fromState, toState = c.updateCongestionState(toState) + shouldNotify = true + } + + if c.congestedCTRTrend != nil && c.congestedCTRTrend.GetDirection() == ccutils.TrendDirectionDownward { + congestedAckedBitrate := c.congestedTrafficStats.AcknowledgedBitrate() + if congestedAckedBitrate < c.estimatedAvailableChannelCapacity { + c.estimatedAvailableChannelCapacity = congestedAckedBitrate + + c.params.Logger.Infow( + "send side bwe: captured traffic ratio is trending downward", + "channel", c.congestedCTRTrend, + "trafficStats", c.congestedTrafficStats, + "estimatedAvailableChannelCapacity", c.estimatedAvailableChannelCapacity, + ) + + shouldNotify = true + } + + // reset to get new set of samples for next trend + c.resetCTRTrend() + } + + return shouldNotify, fromState, toState, c.estimatedAvailableChannelCapacity +} + +func (c *congestionDetector) createCTRTrend() { + c.resetCTRTrend() + c.congestedPacketGroup = nil +} + +func (c *congestionDetector) resetCTRTrend() { + c.congestedCTRTrend = ccutils.NewTrendDetector[float64](ccutils.TrendDetectorParams{ + Name: "ssbwe-ctr", + Logger: c.params.Logger, + Config: c.params.Config.CongestedCTRTrend, + }) + c.congestedTrafficStats = newTrafficStats(trafficStatsParams{ + Config: c.params.Config.WeightedLoss, + Logger: c.params.Logger, + }) +} + +func (c *congestionDetector) clearCTRTrend() { + c.congestedCTRTrend = nil + c.congestedTrafficStats = nil + c.congestedPacketGroup = nil +} + +func (c *congestionDetector) updateCTRTrend(pi *packetInfo, sendDelta, recvDelta int64, isLost bool) { + if c.congestedCTRTrend == nil { + return + } + + if c.congestedPacketGroup == nil { + c.congestedPacketGroup = newPacketGroup( + packetGroupParams{ + Config: c.params.Config.CongestedPacketGroup, + WeightedLoss: c.params.Config.WeightedLoss, + Logger: c.params.Logger, + }, + 0, + ) + } + + if err := c.congestedPacketGroup.Add(pi, sendDelta, recvDelta, isLost); err == errGroupFinalized { + // progressively keep increasing the window and make measurements over longer windows, + // if congestion is not relieving, CTR will trend down + c.congestedTrafficStats.Merge(c.congestedPacketGroup.Traffic()) + + ts := newTrafficStats(trafficStatsParams{ + Config: c.params.Config.WeightedLoss, + Logger: c.params.Logger, + }) + ts.Merge(c.congestedPacketGroup.Traffic()) + ctr := ts.CapturedTrafficRatio() + + // quantise CTR to filter out small changes + c.congestedCTRTrend.AddValue(float64(int((ctr+(c.params.Config.CongestedCTREpsilon/2))/c.params.Config.CongestedCTREpsilon)) * c.params.Config.CongestedCTREpsilon) + + c.congestedPacketGroup = newPacketGroup( + packetGroupParams{ + Config: c.params.Config.CongestedPacketGroup, + WeightedLoss: c.params.Config.WeightedLoss, + Logger: c.params.Logger, + }, + c.congestedPacketGroup.PropagatedQueuingDelay(), + ) + } +} + +func (c *congestionDetector) estimateAvailableChannelCapacity() { + c.estimateTrafficStats = nil + if len(c.packetGroups) == 0 || c.probePacketGroup != nil { + return + } + + // when congested, use contributing groups, + // else use a time windowed measurement + useWindow := false + isAggValid := true + minGroupIdx := 0 + maxGroupIdx := len(c.packetGroups) - 1 + switch c.congestionReason { + case congestionReasonQueuingDelay: + minGroupIdx, maxGroupIdx = c.qdMeasurement.GroupRange() + case congestionReasonLoss: + minGroupIdx, maxGroupIdx = c.lossMeasurement.GroupRange() + default: + useWindow = true + isAggValid = false + } + + agg := newTrafficStats(trafficStatsParams{ + Config: c.params.Config.WeightedLoss, + Logger: c.params.Logger, + }) + for idx := maxGroupIdx; idx >= minGroupIdx; idx-- { + pg := c.packetGroups[idx] + if !pg.IsFinalized() { + continue + } + + agg.Merge(pg.Traffic()) + if useWindow && agg.Duration() > c.params.Config.EstimationWindowDuration.Microseconds() { + isAggValid = true + break + } + } + + if isAggValid { + c.estimatedAvailableChannelCapacity = agg.AcknowledgedBitrate() + c.estimateTrafficStats = agg + } +} + +func (c *congestionDetector) updateCongestionState(state bwe.CongestionState) (bwe.CongestionState, bwe.CongestionState) { + loggingFields := []any{ + "from", c.congestionState, + "to", state, + "congestionReason", c.congestionReason, + "qdMeasurement", c.qdMeasurement, + "lossMeasurement", c.lossMeasurement, + "numPacketGroups", len(c.packetGroups), + "estimatedAvailableChannelCapacity", c.estimatedAvailableChannelCapacity, + "estimateTrafficStats", c.estimateTrafficStats, + } + if c.congestionReason != congestionReasonNone { + var minGroupIdx, maxGroupIdx int + switch c.congestionReason { + case congestionReasonQueuingDelay: + minGroupIdx, maxGroupIdx = c.qdMeasurement.GroupRange() + case congestionReasonLoss: + minGroupIdx, maxGroupIdx = c.lossMeasurement.GroupRange() + } + loggingFields = append( + loggingFields, + "contributingGroups", logger.ObjectSlice(c.packetGroups[minGroupIdx:maxGroupIdx+1]), + ) + } + c.params.Logger.Infow("send side bwe: congestion state change", loggingFields...) + + if state != c.congestionState { + c.congestionStateSwitchedAt = mono.Now() + } + + fromState := c.congestionState + c.congestionState = state + + // when in congested state, monitor changes in captured traffic ratio (CTR) + // to ensure allocations are in line with latest estimates, it is possible that + // the estimate is incorrect when congestion starts and the allocation may be + // sub-optimal and not enough to reduce/relieve congestion, by monitoring CTR + // on a continuous basis allocations can be adjusted in the direction of + // reducing/relieving congestion + if state == bwe.CongestionStateCongested && fromState != bwe.CongestionStateCongested { + c.createCTRTrend() + } else if state != bwe.CongestionStateCongested { + c.clearCTRTrend() + } + + return fromState, c.congestionState +} diff --git a/internal/thirdparty/livekit/sendsidebwe/packet_group.go b/internal/thirdparty/livekit/sendsidebwe/packet_group.go new file mode 100644 index 000000000..b64727fdf --- /dev/null +++ b/internal/thirdparty/livekit/sendsidebwe/packet_group.go @@ -0,0 +1,310 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sendsidebwe + +import ( + "errors" + "time" + + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" + "go.uber.org/zap/zapcore" +) + +// ------------------------------------------------------------- + +var ( + errGroupFinalized = errors.New("packet group is finalized") + errOldPacket = errors.New("packet is older than packet group start") +) + +// ------------------------------------------------------------- + +type PacketGroupConfig struct { + MinPackets int `yaml:"min_packets,omitempty"` + MaxWindowDuration time.Duration `yaml:"max_window_duration,omitempty"` +} + +var ( + defaultPacketGroupConfig = PacketGroupConfig{ + MinPackets: 30, + MaxWindowDuration: 500 * time.Millisecond, + } +) + +// ------------------------------------------------------------- + +type stat struct { + numPackets int + numBytes int +} + +func (s *stat) add(size int) { + s.numPackets++ + s.numBytes += size +} + +func (s *stat) remove(size int) { + s.numPackets-- + s.numBytes -= size +} + +func (s *stat) getNumPackets() int { + return s.numPackets +} + +func (s *stat) getNumBytes() int { + return s.numBytes +} + +func (s stat) MarshalLogObject(e zapcore.ObjectEncoder) error { + e.AddInt("numPackets", s.numPackets) + e.AddInt("numBytes", s.numBytes) + return nil +} + +// ------------------------------------------------------------- + +type classStat struct { + primary stat + rtx stat + probe stat +} + +func (c *classStat) add(size int, isRTX bool, isProbe bool) { + if isRTX { + c.rtx.add(size) + } else if isProbe { + c.probe.add(size) + } else { + c.primary.add(size) + } +} + +func (c *classStat) remove(size int, isRTX bool, isProbe bool) { + if isRTX { + c.rtx.remove(size) + } else if isProbe { + c.probe.remove(size) + } else { + c.primary.remove(size) + } +} + +func (c *classStat) numPackets() int { + return c.primary.getNumPackets() + c.rtx.getNumPackets() + c.probe.getNumPackets() +} + +func (c *classStat) numBytes() int { + return c.primary.getNumBytes() + c.rtx.getNumBytes() + c.probe.getNumBytes() +} + +func (c classStat) MarshalLogObject(e zapcore.ObjectEncoder) error { + e.AddObject("primary", c.primary) + e.AddObject("rtx", c.rtx) + e.AddObject("probe", c.probe) + return nil +} + +// ------------------------------------------------------------- + +type packetGroupParams struct { + Config PacketGroupConfig + WeightedLoss WeightedLossConfig + Logger logger.Logger +} + +type packetGroup struct { + params packetGroupParams + + minSequenceNumber uint64 + maxSequenceNumber uint64 + + minSendTime int64 + maxSendTime int64 + + minRecvTime int64 // for information only + maxRecvTime int64 // for information only + + acked classStat + lost classStat + snBitmap *utils.Bitmap[uint64] + + aggregateSendDelta int64 + aggregateRecvDelta int64 + inheritedQueuingDelay int64 + + isFinalized bool +} + +func newPacketGroup(params packetGroupParams, inheritedQueuingDelay int64) *packetGroup { + return &packetGroup{ + params: params, + inheritedQueuingDelay: inheritedQueuingDelay, + snBitmap: utils.NewBitmap[uint64](params.Config.MinPackets), + } +} + +func (p *packetGroup) Add(pi *packetInfo, sendDelta, recvDelta int64, isLost bool) error { + if isLost { + return p.lostPacket(pi) + } + + if err := p.inGroup(pi.sequenceNumber); err != nil { + return err + } + + if p.minSequenceNumber == 0 || pi.sequenceNumber < p.minSequenceNumber { + p.minSequenceNumber = pi.sequenceNumber + } + p.maxSequenceNumber = max(p.maxSequenceNumber, pi.sequenceNumber) + + if p.minSendTime == 0 || (pi.sendTime-sendDelta) < p.minSendTime { + p.minSendTime = pi.sendTime - sendDelta + } + p.maxSendTime = max(p.maxSendTime, pi.sendTime) + + if p.minRecvTime == 0 || (pi.recvTime-recvDelta) < p.minRecvTime { + p.minRecvTime = pi.recvTime - recvDelta + } + p.maxRecvTime = max(p.maxRecvTime, pi.recvTime) + + p.acked.add(int(pi.size), pi.isRTX, pi.isProbe) + if int(pi.sequenceNumber-p.minSequenceNumber) < p.snBitmap.Len() && p.snBitmap.IsSet(pi.sequenceNumber-p.minSequenceNumber) { + // an earlier packet reported as lost has been received + p.snBitmap.Clear(pi.sequenceNumber - p.minSequenceNumber) + p.lost.remove(int(pi.size), pi.isRTX, pi.isProbe) + } + + // note that out-of-order deliveries will amplify the queueing delay. + // for e.g. a, b, c getting delivered as a, c, b. + // let us say packets are delivered with interval of `x` + // send delta aggregate will go up by x((a, c) = 2x + (c, b) -1x) + // recv delta aggregate will go up by 3x((a, c) = 2x + (c, b) 1x) + p.aggregateSendDelta += sendDelta + p.aggregateRecvDelta += recvDelta + + if p.acked.numPackets() == p.params.Config.MinPackets || (pi.sendTime-p.minSendTime) > p.params.Config.MaxWindowDuration.Microseconds() { + p.isFinalized = true + } + return nil +} + +func (p *packetGroup) lostPacket(pi *packetInfo) error { + if pi.recvTime != 0 { + // previously received packet, so not lost + return nil + } + + if err := p.inGroup(pi.sequenceNumber); err != nil { + return err + } + + if p.minSequenceNumber == 0 || pi.sequenceNumber < p.minSequenceNumber { + p.minSequenceNumber = pi.sequenceNumber + } + p.maxSequenceNumber = max(p.maxSequenceNumber, pi.sequenceNumber) + p.snBitmap.Set(pi.sequenceNumber - p.minSequenceNumber) + + p.lost.add(int(pi.size), pi.isRTX, pi.isProbe) + return nil +} + +func (p *packetGroup) MinSendTime() int64 { + return p.minSendTime +} + +func (p *packetGroup) SendWindow() (int64, int64) { + return p.minSendTime, p.maxSendTime +} + +func (p *packetGroup) PropagatedQueuingDelay() int64 { + if p.inheritedQueuingDelay+p.aggregateRecvDelta-p.aggregateSendDelta > 0 { + return p.inheritedQueuingDelay + p.aggregateRecvDelta - p.aggregateSendDelta + } + + return max(0, p.aggregateRecvDelta-p.aggregateSendDelta) +} + +func (p *packetGroup) FinalizedPropagatedQueuingDelay() (int64, bool) { + if !p.isFinalized { + return 0, false + } + + return p.PropagatedQueuingDelay(), true +} + +func (p *packetGroup) IsFinalized() bool { + return p.isFinalized +} + +func (p *packetGroup) Traffic() *trafficStats { + return &trafficStats{ + minSendTime: p.minSendTime, + maxSendTime: p.maxSendTime, + sendDelta: p.aggregateSendDelta, + recvDelta: p.aggregateRecvDelta, + ackedPackets: p.acked.numPackets(), + ackedBytes: p.acked.numBytes(), + lostPackets: p.lost.numPackets(), + lostBytes: p.lost.numBytes(), + } +} + +func (p *packetGroup) MarshalLogObject(e zapcore.ObjectEncoder) error { + if p == nil { + return nil + } + + e.AddUint64("minSequenceNumber", p.minSequenceNumber) + e.AddUint64("maxSequenceNumber", p.maxSequenceNumber) + e.AddObject("acked", p.acked) + e.AddObject("lost", p.lost) + + e.AddInt64("minRecvTime", p.minRecvTime) + e.AddInt64("maxRecvTime", p.maxRecvTime) + recvDuration := time.Duration((p.maxRecvTime - p.minRecvTime) * 1000) + e.AddDuration("recvDuration", recvDuration) + + recvBitrate := float64(0) + if recvDuration != 0 { + recvBitrate = float64(p.acked.numBytes()*8) / recvDuration.Seconds() + e.AddFloat64("recvBitrate", recvBitrate) + } + + ts := newTrafficStats(trafficStatsParams{ + Config: p.params.WeightedLoss, + Logger: p.params.Logger, + }) + ts.Merge(p.Traffic()) + e.AddObject("trafficStats", ts) + e.AddInt64("inheritedQueuingDelay", p.inheritedQueuingDelay) + e.AddInt64("propagatedQueuingDelay", p.PropagatedQueuingDelay()) + + e.AddBool("isFinalized", p.isFinalized) + return nil +} + +func (p *packetGroup) inGroup(sequenceNumber uint64) error { + if p.isFinalized && sequenceNumber > p.maxSequenceNumber { + return errGroupFinalized + } + + if sequenceNumber < p.minSequenceNumber { + return errOldPacket + } + + return nil +} diff --git a/internal/thirdparty/livekit/sendsidebwe/packet_info.go b/internal/thirdparty/livekit/sendsidebwe/packet_info.go new file mode 100644 index 000000000..730e2a7e6 --- /dev/null +++ b/internal/thirdparty/livekit/sendsidebwe/packet_info.go @@ -0,0 +1,45 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sendsidebwe + +import ( + "github.com/livekit/livekit-server/pkg/sfu/ccutils" + "go.uber.org/zap/zapcore" +) + +type packetInfo struct { + sequenceNumber uint64 + sendTime int64 + recvTime int64 + probeClusterId ccutils.ProbeClusterId + size uint16 + isRTX bool + isProbe bool +} + +func (pi *packetInfo) MarshalLogObject(e zapcore.ObjectEncoder) error { + if pi == nil { + return nil + } + + e.AddUint64("sequenceNumber", pi.sequenceNumber) + e.AddInt64("sendTime", pi.sendTime) + e.AddInt64("recvTime", pi.recvTime) + e.AddUint32("probeClusterId", uint32(pi.probeClusterId)) + e.AddUint16("size", pi.size) + e.AddBool("isRTX", pi.isRTX) + e.AddBool("isProbe", pi.isProbe) + return nil +} diff --git a/internal/thirdparty/livekit/sendsidebwe/packet_tracker.go b/internal/thirdparty/livekit/sendsidebwe/packet_tracker.go new file mode 100644 index 000000000..a41e50fee --- /dev/null +++ b/internal/thirdparty/livekit/sendsidebwe/packet_tracker.go @@ -0,0 +1,168 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sendsidebwe + +import ( + "math/rand" + "sync" + + "github.com/livekit/livekit-server/pkg/sfu/ccutils" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils/mono" +) + +// ------------------------------------------------------------------------------- + +type packetTrackerParams struct { + Logger logger.Logger +} + +type packetTracker struct { + params packetTrackerParams + + lock sync.Mutex + + sequenceNumber uint64 + + baseSendTime int64 + packetInfos [2048]packetInfo + + baseRecvTime int64 + piLastRecv *packetInfo + + probeClusterId ccutils.ProbeClusterId + probeMaxSequenceNumber uint64 +} + +func newPacketTracker(params packetTrackerParams) *packetTracker { + return &packetTracker{ + params: params, + sequenceNumber: uint64(rand.Intn(1<<14)) + uint64(1<<15), // a random number in third quartile of sequence number space + } +} + +func (p *packetTracker) RecordPacketSendAndGetSequenceNumber( + atMicro int64, + size int, + isRTX bool, + probeClusterId ccutils.ProbeClusterId, + isProbe bool, +) uint16 { + p.lock.Lock() + defer p.lock.Unlock() + + if p.baseSendTime == 0 { + p.baseSendTime = atMicro + } + + pi := p.getPacketInfo(uint16(p.sequenceNumber)) + *pi = packetInfo{ + sequenceNumber: p.sequenceNumber, + sendTime: atMicro - p.baseSendTime, + size: uint16(size), + isRTX: isRTX, + probeClusterId: probeClusterId, + isProbe: isProbe, + } + + p.sequenceNumber++ + + // extreme case of wrap around before receiving any feedback + if pi == p.piLastRecv { + p.piLastRecv = nil + } + + if p.probeClusterId != ccutils.ProbeClusterIdInvalid && p.probeClusterId == pi.probeClusterId && pi.sequenceNumber > p.probeMaxSequenceNumber { + p.probeMaxSequenceNumber = pi.sequenceNumber + } + + return uint16(pi.sequenceNumber) +} + +func (p *packetTracker) BaseSendTimeThreshold(threshold int64) (int64, bool) { + p.lock.Lock() + defer p.lock.Unlock() + + if p.baseSendTime == 0 { + return 0, false + } + + return mono.UnixMicro() - p.baseSendTime - threshold, true +} + +func (p *packetTracker) RecordPacketIndicationFromRemote(sn uint16, recvTime int64) (piRecv packetInfo, sendDelta, recvDelta int64) { + p.lock.Lock() + defer p.lock.Unlock() + + pi := p.getPacketInfoExisting(sn) + if pi == nil { + return + } + + if recvTime == 0 { + // maybe lost OR already received but reported lost in a later report + piRecv = *pi + return + } + + if p.baseRecvTime == 0 { + p.baseRecvTime = recvTime + p.piLastRecv = pi + } + + pi.recvTime = recvTime - p.baseRecvTime + piRecv = *pi + if p.piLastRecv != nil { + sendDelta, recvDelta = pi.sendTime-p.piLastRecv.sendTime, pi.recvTime-p.piLastRecv.recvTime + } + p.piLastRecv = pi + return +} + +func (p *packetTracker) getPacketInfo(sn uint16) *packetInfo { + return &p.packetInfos[int(sn)%len(p.packetInfos)] +} + +func (p *packetTracker) getPacketInfoExisting(sn uint16) *packetInfo { + pi := &p.packetInfos[int(sn)%len(p.packetInfos)] + if uint16(pi.sequenceNumber) == sn { + return pi + } + + return nil +} + +func (p *packetTracker) ProbeClusterStarting(probeClusterId ccutils.ProbeClusterId) { + p.lock.Lock() + defer p.lock.Unlock() + + p.probeClusterId = probeClusterId +} + +func (p *packetTracker) ProbeClusterDone(probeClusterId ccutils.ProbeClusterId) { + p.lock.Lock() + defer p.lock.Unlock() + + if p.probeClusterId == probeClusterId { + p.probeClusterId = ccutils.ProbeClusterIdInvalid + } +} + +func (p *packetTracker) ProbeMaxSequenceNumber() uint64 { + p.lock.Lock() + defer p.lock.Unlock() + + return p.probeMaxSequenceNumber +} diff --git a/internal/thirdparty/livekit/sendsidebwe/probe_packet_group.go b/internal/thirdparty/livekit/sendsidebwe/probe_packet_group.go new file mode 100644 index 000000000..8d0d1d72a --- /dev/null +++ b/internal/thirdparty/livekit/sendsidebwe/probe_packet_group.go @@ -0,0 +1,143 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Modified by Piik: start probe timing at its first acknowledged packet. +package sendsidebwe + +import ( + "time" + + "github.com/livekit/livekit-server/pkg/sfu/ccutils" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils/mono" + "go.uber.org/zap/zapcore" +) + +// ------------------------------------------------------------- + +type ProbePacketGroupConfig struct { + PacketGroup PacketGroupConfig `yaml:"packet_group,omitempty"` + + SettleWaitNumRTT uint32 `yaml:"settle_wait_num_rtt,omitempty"` + SettleWaitMin time.Duration `yaml:"settle_wait_min,omitempty"` + SettleWaitMax time.Duration `yaml:"settle_wait_max,omitempty"` +} + +var ( + // large numbers to treat a probe packet group as one + defaultProbePacketGroupConfig = ProbePacketGroupConfig{ + PacketGroup: PacketGroupConfig{ + MinPackets: 16384, + MaxWindowDuration: time.Minute, + }, + + SettleWaitNumRTT: 5, + SettleWaitMin: 250 * time.Millisecond, + SettleWaitMax: 5 * time.Second, + } +) + +// ------------------------------------------------------------- + +type probePacketGroupParams struct { + Config ProbePacketGroupConfig + WeightedLoss WeightedLossConfig + Logger logger.Logger +} + +type probePacketGroup struct { + params probePacketGroupParams + pci ccutils.ProbeClusterInfo + *packetGroup + maxSequenceNumber uint64 + doneAt time.Time +} + +func newProbePacketGroup(params probePacketGroupParams, pci ccutils.ProbeClusterInfo) *probePacketGroup { + return &probePacketGroup{ + params: params, + pci: pci, + packetGroup: newPacketGroup( + packetGroupParams{ + Config: params.Config.PacketGroup, + WeightedLoss: params.WeightedLoss, + Logger: params.Logger, + }, + 0, + ), + } +} + +func (p *probePacketGroup) ProbeClusterDone(pci ccutils.ProbeClusterInfo) { + if p.pci.Id != pci.Id { + return + } + + p.pci.Result = pci.Result + p.doneAt = mono.Now() +} + +func (p *probePacketGroup) ProbeClusterInfo() ccutils.ProbeClusterInfo { + return p.pci +} + +func (p *probePacketGroup) MaybeFinalizeProbe(maxSequenceNumber uint64, rtt float64) (ccutils.ProbeClusterInfo, bool) { + if p.doneAt.IsZero() { + return ccutils.ProbeClusterInfoInvalid, false + } + + if maxSequenceNumber != 0 && p.maxSequenceNumber >= maxSequenceNumber { + return p.pci, true + } + + settleWait := time.Duration(float64(p.params.Config.SettleWaitNumRTT) * rtt * float64(time.Second)) + if settleWait < p.params.Config.SettleWaitMin { + settleWait = p.params.Config.SettleWaitMin + } + if settleWait > p.params.Config.SettleWaitMax { + settleWait = p.params.Config.SettleWaitMax + } + if time.Since(p.doneAt) < settleWait { + return ccutils.ProbeClusterInfoInvalid, false + } + + return p.pci, true +} + +func (p *probePacketGroup) Add(pi *packetInfo, sendDelta, recvDelta int64, isLost bool) error { + if pi.probeClusterId != p.pci.Id { + return nil + } + + p.maxSequenceNumber = max(p.maxSequenceNumber, pi.sequenceNumber) + + // The first received probe packet starts this measurement window. + // The transport's previous packet may precede it by an idle interval. + if !isLost && p.acked.numPackets() == 0 { + sendDelta, recvDelta = 0, 0 + } + return p.packetGroup.Add(pi, sendDelta, recvDelta, isLost) +} + +func (p *probePacketGroup) MarshalLogObject(e zapcore.ObjectEncoder) error { + if p == nil { + return nil + } + + e.AddObject("pci", p.pci) + e.AddObject("packetGroup", p.packetGroup) + e.AddUint64("maxSequenceNumber", p.maxSequenceNumber) + e.AddTime("doneAt", p.doneAt) + return nil +} diff --git a/internal/thirdparty/livekit/sendsidebwe/send_side_bwe.go b/internal/thirdparty/livekit/sendsidebwe/send_side_bwe.go new file mode 100644 index 000000000..df631f27e --- /dev/null +++ b/internal/thirdparty/livekit/sendsidebwe/send_side_bwe.go @@ -0,0 +1,151 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sendsidebwe + +import ( + "time" + + "github.com/livekit/livekit-server/pkg/sfu/bwe" + "github.com/livekit/livekit-server/pkg/sfu/ccutils" + "github.com/livekit/protocol/logger" + "github.com/pion/rtcp" +) + +var _ bwe.BWE = (*SendSideBWE)(nil) + +// +// Based on a simplified/modified version of JitterPath paper +// (https://homepage.iis.sinica.edu.tw/papers/lcs/2114-F.pdf) +// +// TWCC feedback is used to calculate delta one-way-delay. +// It is accumulated/propagated to determine in which region +// groups of packets are operating in. +// +// In simplified terms, +// o JQR (Join Queuing Region) is when channel is congested. +// o DQR (Disjoint Queuing Region) is when channel is not. +// +// Packets are grouped and thresholds applied to smooth over +// small variations. For example, in the paper, +// if propagated_queuing_delay + delta_one_way_delay > 0 { +// possibly_operating_in_jqr +// } +// But, in this implementation it is checked at packet group level, +// i. e. using queuing delay and aggregated delta one-way-delay of +// the group and a minimum value threshold is applied before declaring +// that a group is in JQR. +// +// There is also hysteresis to make transisitons smoother, i.e. if the +// metric is above a certain threshold, it is JQR and it is DQR only if it +// is below a certain value and the gap in between those two thresholds +// are treated as interdeterminate groups. +// + +// --------------------------------------------------------------------------- + +type SendSideBWEConfig struct { + CongestionDetector CongestionDetectorConfig `yaml:"congestion_detector,omitempty"` +} + +var ( + DefaultSendSideBWEConfig = SendSideBWEConfig{ + CongestionDetector: defaultCongestionDetectorConfig, + } +) + +// --------------------------------------------------------------------------- + +type SendSideBWEParams struct { + Config SendSideBWEConfig + Logger logger.Logger +} + +type SendSideBWE struct { + bwe.NullBWE + + params SendSideBWEParams + + *congestionDetector +} + +func NewSendSideBWE(params SendSideBWEParams) *SendSideBWE { + return &SendSideBWE{ + params: params, + congestionDetector: newCongestionDetector(congestionDetectorParams{ + Config: params.Config.CongestionDetector, + Logger: params.Logger, + }), + } +} + +func (s *SendSideBWE) Type() bwe.BWEType { + return bwe.BWETypeSendSide +} + +func (s *SendSideBWE) SetBWEListener(bweListener bwe.BWEListener) { + s.congestionDetector.SetBWEListener(bweListener) +} + +func (s *SendSideBWE) Reset() { + s.congestionDetector.Reset() +} + +func (s *SendSideBWE) RecordPacketSendAndGetSequenceNumber( + atMicro int64, + size int, + isRTX bool, + probeClusterId ccutils.ProbeClusterId, + isProbe bool, +) uint16 { + return s.congestionDetector.RecordPacketSendAndGetSequenceNumber(atMicro, size, isRTX, probeClusterId, isProbe) +} + +func (s *SendSideBWE) HandleTWCCFeedback(report *rtcp.TransportLayerCC) { + s.congestionDetector.HandleTWCCFeedback(report) +} + +func (s *SendSideBWE) UpdateRTT(rtt float64) { + s.congestionDetector.UpdateRTT(rtt) +} + +func (s *SendSideBWE) CongestionState() bwe.CongestionState { + return s.congestionDetector.CongestionState() +} + +func (s *SendSideBWE) CanProbe() bool { + return s.congestionDetector.CanProbe() +} + +func (s *SendSideBWE) ProbeDuration() time.Duration { + return s.congestionDetector.ProbeDuration() +} + +func (s *SendSideBWE) ProbeClusterStarting(pci ccutils.ProbeClusterInfo) { + s.congestionDetector.ProbeClusterStarting(pci) +} + +func (s *SendSideBWE) ProbeClusterDone(pci ccutils.ProbeClusterInfo) { + s.congestionDetector.ProbeClusterDone(pci) +} + +func (s *SendSideBWE) ProbeClusterIsGoalReached() bool { + return s.congestionDetector.ProbeClusterIsGoalReached() +} + +func (s *SendSideBWE) ProbeClusterFinalize() (ccutils.ProbeSignal, int64, bool) { + return s.congestionDetector.ProbeClusterFinalize() +} + +// ------------------------------------------------ diff --git a/internal/thirdparty/livekit/sendsidebwe/traffic_stats.go b/internal/thirdparty/livekit/sendsidebwe/traffic_stats.go new file mode 100644 index 000000000..f1c07d54e --- /dev/null +++ b/internal/thirdparty/livekit/sendsidebwe/traffic_stats.go @@ -0,0 +1,191 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sendsidebwe + +import ( + "math" + "time" + + "github.com/livekit/protocol/logger" + "go.uber.org/zap/zapcore" +) + +// ----------------------------------------------------------- + +type WeightedLossConfig struct { + MinDurationForLossValidity time.Duration `yaml:"min_duration_for_loss_validity,omitempty"` + BaseDuration time.Duration `yaml:"base_duration,omitempty"` + BasePPS int `yaml:"base_pps,omitempty"` + LossPenaltyFactor float64 `yaml:"loss_penalty_factor,omitempty"` +} + +var ( + defaultWeightedLossConfig = WeightedLossConfig{ + MinDurationForLossValidity: 100 * time.Millisecond, + BaseDuration: 500 * time.Millisecond, + BasePPS: 30, + LossPenaltyFactor: 0.25, + } +) + +// ----------------------------------------------------------- + +type trafficStatsParams struct { + Config WeightedLossConfig + Logger logger.Logger +} + +type trafficStats struct { + params trafficStatsParams + + minSendTime int64 + maxSendTime int64 + sendDelta int64 + recvDelta int64 + ackedPackets int + ackedBytes int + lostPackets int + lostBytes int +} + +func newTrafficStats(params trafficStatsParams) *trafficStats { + return &trafficStats{ + params: params, + } +} + +func (ts *trafficStats) Merge(rhs *trafficStats) { + if ts.minSendTime == 0 || rhs.minSendTime < ts.minSendTime { + ts.minSendTime = rhs.minSendTime + } + if rhs.maxSendTime > ts.maxSendTime { + ts.maxSendTime = rhs.maxSendTime + } + ts.sendDelta += rhs.sendDelta + ts.recvDelta += rhs.recvDelta + ts.ackedPackets += rhs.ackedPackets + ts.ackedBytes += rhs.ackedBytes + ts.lostPackets += rhs.lostPackets + ts.lostBytes += rhs.lostBytes +} + +func (ts *trafficStats) NumBytes() int { + return ts.ackedBytes + ts.lostBytes +} + +func (ts *trafficStats) Duration() int64 { + return ts.maxSendTime - ts.minSendTime +} + +func (ts *trafficStats) AcknowledgedBitrate() int64 { + duration := ts.Duration() + if duration == 0 { + return 0 + } + + ackedBitrate := float64(ts.ackedBytes) * 8 * 1e6 / float64(ts.Duration()) + return int64(ackedBitrate * ts.CapturedTrafficRatio()) +} + +func (ts *trafficStats) CapturedTrafficRatio() float64 { + if ts.recvDelta == 0 { + return 0.0 + } + + // apply a penalty for lost packets, + // the rationale being packet dropping is a strategy to relieve congestion + // and if they were not dropped, they would have increased queuing delay, + // as it is not possible to know the reason for the losses, + // apply a small penalty to receive delta aggregate to simulate those packets + // building up queuing delay. + return min(1.0, float64(ts.sendDelta)/float64(ts.recvDelta+ts.lossPenalty())) +} + +func (ts *trafficStats) WeightedLoss() float64 { + durationMicro := ts.Duration() + if time.Duration(durationMicro*1000) < ts.params.Config.MinDurationForLossValidity { + return 0.0 + } + + totalPackets := float64(ts.lostPackets + ts.ackedPackets) + pps := totalPackets * 1e6 / float64(durationMicro) + + // longer duration, i. e. more time resolution, lower pps is acceptable as the measurement is more stable + deltaDuration := time.Duration(durationMicro*1000) - ts.params.Config.BaseDuration + if deltaDuration < 0 { + deltaDuration = 0 + } + threshold := math.Exp(-deltaDuration.Seconds()) * float64(ts.params.Config.BasePPS) + if pps < threshold { + return 0.0 + } + + lossRatio := float64(0.0) + if totalPackets != 0 { + lossRatio = float64(ts.lostPackets) / totalPackets + } + + // Log10 is used to give higher weight for the same loss ratio at higher packet rates, + // for e.g. + // - 10% loss at 20 pps = 0.1 * log10(20) = 0.130 + // - 10% loss at 100 pps = 0.1 * log10(100) = 0.2 + // - 10% loss at 1000 pps = 0.1 * log10(1000) = 0.3 + return lossRatio * math.Log10(pps) +} + +func (ts *trafficStats) lossPenalty() int64 { + return int64(float64(ts.recvDelta) * ts.WeightedLoss() * ts.params.Config.LossPenaltyFactor) +} + +func (ts *trafficStats) MarshalLogObject(e zapcore.ObjectEncoder) error { + if ts == nil { + return nil + } + + e.AddInt64("minSendTime", ts.minSendTime) + e.AddInt64("maxSendTime", ts.maxSendTime) + duration := time.Duration(ts.Duration() * 1000) + e.AddDuration("duration", duration) + + e.AddInt("ackedPackets", ts.ackedPackets) + e.AddInt("ackedBytes", ts.ackedBytes) + e.AddInt("lostPackets", ts.lostPackets) + e.AddInt("lostBytes", ts.lostBytes) + + bitrate := float64(0) + if duration != 0 { + bitrate = float64(ts.ackedBytes*8) / duration.Seconds() + e.AddFloat64("bitrate", bitrate) + } + + e.AddInt64("sendDelta", ts.sendDelta) + e.AddInt64("recvDelta", ts.recvDelta) + e.AddInt64("groupDelay", ts.recvDelta-ts.sendDelta) + + totalPackets := ts.lostPackets + ts.ackedPackets + if duration != 0 { + e.AddFloat64("pps", float64(totalPackets)/duration.Seconds()) + } + if (totalPackets) != 0 { + e.AddFloat64("rawLoss", float64(ts.lostPackets)/float64(totalPackets)) + } + e.AddFloat64("weightedLoss", ts.WeightedLoss()) + e.AddInt64("lossPenalty", ts.lossPenalty()) + + capturedTrafficRatio := ts.CapturedTrafficRatio() + e.AddFloat64("capturedTrafficRatio", capturedTrafficRatio) + e.AddFloat64("estimatedAvailableChannelCapacity", bitrate*capturedTrafficRatio) + return nil +} diff --git a/internal/thirdparty/livekit/sendsidebwe/twcc_feedback.go b/internal/thirdparty/livekit/sendsidebwe/twcc_feedback.go new file mode 100644 index 000000000..2b2c843e8 --- /dev/null +++ b/internal/thirdparty/livekit/sendsidebwe/twcc_feedback.go @@ -0,0 +1,127 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sendsidebwe + +import ( + "time" + + "github.com/livekit/protocol/logger" + "github.com/pion/rtcp" + "go.uber.org/zap/zapcore" +) + +// ------------------------------------------------------ + +const ( + cOutlierReportFactor = 3 + cEstimatedFeedbackIntervalAlpha = float64(0.9) + + cReferenceTimeMask = (1 << 24) - 1 + cReferenceTimeResolution = 64 // 64 ms +) + +// ------------------------------------------------------ + +type twccFeedbackParams struct { + Logger logger.Logger +} + +type twccFeedback struct { + params twccFeedbackParams + + lastFeedbackTime time.Time + estimatedFeedbackInterval time.Duration + numReports int + numReportsOutOfOrder int + + highestFeedbackCount uint8 + + cycles int64 + highestReferenceTime uint32 +} + +func newTWCCFeedback(params twccFeedbackParams) *twccFeedback { + return &twccFeedback{ + params: params, + } +} + +func (t *twccFeedback) ProcessReport(report *rtcp.TransportLayerCC, at time.Time) (int64, bool) { + t.numReports++ + if t.lastFeedbackTime.IsZero() { + t.lastFeedbackTime = at + t.highestReferenceTime = report.ReferenceTime + t.highestFeedbackCount = report.FbPktCount + return (t.cycles + int64(report.ReferenceTime)) * cReferenceTimeResolution * 1000, false + } + + isOutOfOrder := false + if (report.FbPktCount - t.highestFeedbackCount) > (1 << 7) { + t.numReportsOutOfOrder++ + isOutOfOrder = true + t.params.Logger.Infow( + "send side bwe: received out-of-order feedback report", + "highestFeedbackCount", t.highestFeedbackCount, + "fbPktCount", report.FbPktCount, + "numReportsOutOfOrder", t.numReportsOutOfOrder, + ) + } + + // reference time wrap around handling + var referenceTime int64 + if (report.ReferenceTime-t.highestReferenceTime)&cReferenceTimeMask < (1 << 23) { + if report.ReferenceTime < t.highestReferenceTime { + t.cycles += (1 << 24) + } + t.highestReferenceTime = report.ReferenceTime + referenceTime = t.cycles + int64(report.ReferenceTime) + } else { + cycles := t.cycles + if report.ReferenceTime > t.highestReferenceTime && cycles >= (1<<24) { + cycles -= (1 << 24) + } + referenceTime = cycles + int64(report.ReferenceTime) + } + + if !isOutOfOrder { + sinceLast := at.Sub(t.lastFeedbackTime) + if t.estimatedFeedbackInterval == 0 { + t.estimatedFeedbackInterval = sinceLast + } else { + // filter out outliers from estimate + if sinceLast > t.estimatedFeedbackInterval/cOutlierReportFactor && sinceLast < cOutlierReportFactor*t.estimatedFeedbackInterval { + // smoothed version of inter feedback interval + t.estimatedFeedbackInterval = time.Duration(cEstimatedFeedbackIntervalAlpha*float64(t.estimatedFeedbackInterval) + (1.0-cEstimatedFeedbackIntervalAlpha)*float64(sinceLast)) + } + } + t.lastFeedbackTime = at + t.highestFeedbackCount = report.FbPktCount + } + + return referenceTime * cReferenceTimeResolution * 1000, isOutOfOrder +} + +func (t *twccFeedback) MarshalLogObject(e zapcore.ObjectEncoder) error { + if t == nil { + return nil + } + + e.AddTime("lastFeedbackTime", t.lastFeedbackTime) + e.AddDuration("estimatedFeedbackInterval", t.estimatedFeedbackInterval) + e.AddInt("numReports", t.numReports) + e.AddInt("numReportsOutOfOrder", t.numReportsOutOfOrder) + e.AddInt64("cycles", t.cycles/(1<<24)) + return nil +} From 76bfa1a0ec6591ac29b666cae87a1ef25a1cab23 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:12:14 +0800 Subject: [PATCH 087/262] fix(media): keep bandwidth accounting scoped to video --- docs/research/browser-screen-audio-quality.md | 9 +- docs/research/node-local-media-design.md | 28 +++-- docs/todo.md | 5 +- internal/app/mediaedge/audio.go | 14 +-- internal/app/mediaedge/edge.go | 1 - internal/app/mediaedge/publication.go | 2 - internal/app/mediaedge/quality.go | 10 +- internal/app/mediaedge/quality_test.go | 6 +- internal/media/forwarding/output.go | 2 +- internal/media/forwarding/publication.go | 15 +-- internal/media/forwarding/publication_test.go | 36 +++--- internal/media/forwarding/recovery_test.go | 117 ++++++++++++++++-- internal/media/forwarding/transport.go | 17 +-- internal/media/forwarding/transport_test.go | 5 +- internal/server/sfu/media.go | 12 -- 15 files changed, 168 insertions(+), 111 deletions(-) diff --git a/docs/research/browser-screen-audio-quality.md b/docs/research/browser-screen-audio-quality.md index e51a21eab..b45c08c7d 100644 --- a/docs/research/browser-screen-audio-quality.md +++ b/docs/research/browser-screen-audio-quality.md @@ -107,11 +107,10 @@ advancing decoded audio and successful cleanup. A direct Browser control now keeps requested 256/320 kbps below an explicit 192 kbps receive limit, while a 510 kbps receiver permits both rates. A native real-Opus control through 6 Mbps -> 500 kbps -> 6 Mbps shaping retains decoded audio during live -320/96/320 kbps changes. It exposed the shared -[probe measurement repairs](./node-local-media-design.md#send-queue-recovery), -but video allocation after returning to the high audio rate still needs -verification. These are local runtime checks; physical networks, mobile playback -and listening remain open. +320/96/320 kbps changes, with sustained high video after the shared +[probe measurement and video-budget repairs](./node-local-media-design.md#send-queue-recovery). +These are local runtime checks; physical networks, mobile playback and listening +remain open. RFC 7587 gives an omitted receive bitrate a mode-dependent default, not an unrestricted value, and asks receivers to ignore out-of-range preferences. diff --git a/docs/research/node-local-media-design.md b/docs/research/node-local-media-design.md index 4da1093a4..18589d465 100644 --- a/docs/research/node-local-media-design.md +++ b/docs/research/node-local-media-design.md @@ -223,8 +223,8 @@ input anchor, not independently to each encoder's completion time. Local capture replacement keeps its RTP source and cumulative counters while rebasing only the input packetizers. It must not also restart the RTP projector. -Unconnected edges reserve capacity without asking for output. Existing audio -settings or observed relay-audio bytes reserve that portion of the PC budget. +Unconnected edges reserve child slots without asking for output. Video capacity +comes from its tracked traffic; untracked audio must not be subtracted again. Capture producers now implement the coordinated v5 multi-output boundary below; the packaged runtime has not been physically accepted. A bounded real libvpx @@ -411,8 +411,8 @@ owns the decoded-picture and address-change boundary. Congestion callbacks could also promote the estimator's unmeasured 100 Mbps initial capacity while a probe was pending. Congestion now only reduces the committed budget; a valid completed non-congesting probe owns increases. -Publication allocation and encoder input use that same committed capacity after -audio reservation. This follows the +Publication allocation and encoder input use that same committed video capacity. +This follows the [upstream distinction between congestion events and completed probes](https://github.com/livekit/livekit/blob/v1.13.6/pkg/sfu/streamallocator/streamallocator.go), without introducing another estimator. @@ -449,10 +449,22 @@ could promote 100 Mbps and do not establish correct budgeting. The SDK latency-aware queue requires an extra adapter and is not retained. Earlier flaky comparisons also registered TWCC twice on the receiver, duplicating feedback; those results do not rank the queues. Product receivers already use -the single default interceptor owner. A separate real Opus 320/96/320 kbps -control retains audio decoding under the same impairment; video can still -return to a lower output after the high audio rate resumes. That allocation -boundary remains open rather than being reported as complete recovery. +the single default interceptor owner. + +Live audio changes exposed a related scope mismatch: Piik's TWCC tracks only +video, but the adapter deducted audio from that measured capacity and included +audio in expected probe usage. After restoring 320 kbps Opus, this could remove +an already recovered video layer on a healthy 6 Mbps link. The adapter now uses +the measured video capacity directly, matching the upstream allocator's +video-only usage accounting. Native and SFU share this rule; quality reports +compare video capacity with video payload while retaining total traffic in +diagnostics. Opus rate and transport behavior are unchanged. The retained +network gate exercises video-only and real Opus 320/96/320 kbps cases, requiring +sustained high video and decoded audio after recovery. The former subtraction +fails a matched trace but can be masked by later probe traffic; a passing +recovery alone does not establish correct accounting. This is not proof of +perceptual audio quality or behavior on every physical network. + No reset, retry owner or minimum quality is added. These are bounded local traces, not a field diagnosis, success-rate or game-latency claim. Ordinary focused checks cover NACK repair, no-RR probe admission, zero-write diff --git a/docs/todo.md b/docs/todo.md index 53052b009..f23a4bae3 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -57,9 +57,8 @@ Implementation needs a concrete useful result within the product model. limits and framework-owned adaptation. The [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment) retain explicit receive-limit enforcement and the current 320 kbps sender - ceiling. A real-Opus impaired-network control retains decoding; verify video - allocation after restoring high audio rates. Physical listening, mobile and - field-network acceptance remain open. + ceiling. Physical listening, mobile and field-network acceptance remain open; + research owns the bounded real-Opus impairment and video-recovery control. New codecs need complete encode/decode/forward support and a late-Viewer policy before inclusion; selecting a codec cannot transcode an existing publication. Confirm live diff --git a/internal/app/mediaedge/audio.go b/internal/app/mediaedge/audio.go index d658dfbfe..a3b840507 100644 --- a/internal/app/mediaedge/audio.go +++ b/internal/app/mediaedge/audio.go @@ -147,19 +147,7 @@ func (source *AudioSource) SetBitrate(bitrate int) error { if closed { return errors.New("native audio source is closed") } - if err := source.encoder.SetBitrate(bitrate); err != nil { - return err - } - source.mu.Lock() - edges := make([]*Edge, 0, len(source.edges)) - for edge := range source.edges { - edges = append(edges, edge) - } - source.mu.Unlock() - for _, edge := range edges { - edge.transport.SetAudioBitrate(uint32(bitrate)) - } - return nil + return source.encoder.SetBitrate(bitrate) } func (source *AudioSource) configuredBitrate() uint32 { diff --git a/internal/app/mediaedge/edge.go b/internal/app/mediaedge/edge.go index 16bd3dc24..ea79d8923 100644 --- a/internal/app/mediaedge/edge.go +++ b/internal/app/mediaedge/edge.go @@ -105,7 +105,6 @@ func (engine *Engine) NewEdge(source *Source, options EdgeOptions) (_ *Edge, err } connection := transport.PC log := slog.Default().With("connectionId", diagnostics.ID(options.ConnectionID), "rtcPeerId", diagnostics.ID(connection.ID())) - transport.SetAudioBitrate(options.Audio.configuredBitrate()) edge := &Edge{ connectionID: options.ConnectionID, engine: engine, diff --git a/internal/app/mediaedge/publication.go b/internal/app/mediaedge/publication.go index a860e08c6..11ad63ba3 100644 --- a/internal/app/mediaedge/publication.go +++ b/internal/app/mediaedge/publication.go @@ -82,7 +82,6 @@ func (engine *Engine) NewPublication(source *Source, options EdgeOptions) (*Publ return nil, err } publication.transport = transport - transport.SetAudioBitrate(options.Audio.configuredBitrate()) publication.signaling.connection = transport.PC publication.signaling.audioSource = options.Audio publication.signaling.localCandidates = newLocalCandidateGathering(socket, options.ICEServers, nil, options.Events.LocalCandidate) @@ -208,7 +207,6 @@ func (publication *Publication) Media() PublicationMedia { media.Layers[index] = PublicationLayer{RID: forwarding.PublicationRID(index, len(info.Layers)), Width: layer.Width, Height: layer.Height, Bitrate: layer.Bitrate} } - publication.transport.SetAudioBitrate(media.AudioBitrate) return media } diff --git a/internal/app/mediaedge/quality.go b/internal/app/mediaedge/quality.go index f164c0573..354c24b74 100644 --- a/internal/app/mediaedge/quality.go +++ b/internal/app/mediaedge/quality.go @@ -52,11 +52,9 @@ func (edge *Edge) QualitySample(now time.Time) (QualitySample, bool) { return QualitySample{}, false } frames := framesNow - previous.frames - bytes := bytesNow - previous.videoBytes + audioBytes - previous.audioBytes + videoBytes := bytesNow - previous.videoBytes + bytes := videoBytes + audioBytes - previous.audioBytes seconds := window.Seconds() - if edge.audioSource != nil && edge.audioSource.encoder == nil { - edge.transport.SetAudioBitrate(uint32(float64((audioBytes-previous.audioBytes)*8) / seconds)) - } bitrate := float64(bytes*8) / seconds sample := QualitySample{ SampleTimestampMs: now.UnixMilli(), @@ -87,7 +85,9 @@ func (edge *Edge) QualitySample(now time.Time) (QualitySample, bool) { sample.AvailableOutgoingKbps = float64(target) / 1000 reason := "none" sample.State = "healthy" - if edge.transport.Output.IsDeficient() || float64(target) < bitrate { + // The estimator tracks video only. Keep total traffic in diagnostics, but + // compare its capacity with the corresponding video payload. + if edge.transport.Output.IsDeficient() || float64(target) < float64(videoBytes*8)/seconds { reason = "bandwidth" sample.State = "degraded" } diff --git a/internal/app/mediaedge/quality_test.go b/internal/app/mediaedge/quality_test.go index 971c8d634..7e225ade1 100644 --- a/internal/app/mediaedge/quality_test.go +++ b/internal/app/mediaedge/quality_test.go @@ -105,6 +105,7 @@ func TestQualitySampleUsesEstimatorCapacityAgainstEncodedPayload(t *testing.T) { output.UpTrackMaxPublishedLayerChange(1) output.UpTrackMaxTemporalLayerSeenChange(0) output.SetMaxSpatialLayer(1) + edge.audioSource = &AudioSource{} for index, check := range []struct { budget, sent int layer int32 @@ -128,8 +129,11 @@ func TestQualitySampleUsesEstimatorCapacityAgainstEncodedPayload(t *testing.T) { stats.Update(started.Add(time.Duration(number)*time.Second/30).UnixNano(), number, number*3000, true, 12, size, 0, false) } + // Audio remains in total traffic diagnostics, but cannot turn a healthy + // 900 kbps video into an apparent overrun of its 1 Mbps video estimate. + edge.audioSource.bytes.Add(80_000) // 320 kbps during this two-second window. sample, ok = edge.QualitySample(started.Add(time.Duration(index+2) * 2 * time.Second)) - if !ok || sample.State != check.state || sample.BitrateKbps != float64(check.sent)/1000 { + if !ok || sample.State != check.state || sample.BitrateKbps != float64(check.sent)/1000+320 { t.Fatalf("allocation %d quality = %+v, %v", index, sample, ok) } } diff --git a/internal/media/forwarding/output.go b/internal/media/forwarding/output.go index 000f92b77..469afc7a4 100644 --- a/internal/media/forwarding/output.go +++ b/internal/media/forwarding/output.go @@ -38,7 +38,7 @@ type Output struct { type Snapshot struct { Prepare, Target, Requested, Current int32 Paused bool - // VideoBudget is the current connection estimate after audio reservation. + // VideoBudget is the committed estimate for this connection's tracked video. // It is codec input, not a claim that this bitrate is available or delivered. VideoBudget int64 } diff --git a/internal/media/forwarding/publication.go b/internal/media/forwarding/publication.go index 5404dec6a..08537672a 100644 --- a/internal/media/forwarding/publication.go +++ b/internal/media/forwarding/publication.go @@ -292,11 +292,8 @@ func (publication *Publication) RequiredActiveCount() int { return publication.activeCount } -func (publication *Publication) SetAudioBitrate(bitrate uint32) { - publication.bandwidth.audio.Store(int64(bitrate)) -} func (publication *Publication) VideoBudget() int64 { - return max(publication.bandwidth.target.Load()-publication.bandwidth.audio.Load(), 0) + return publication.bandwidth.target.Load() } func (publication *Publication) LowestLayerBudget() int64 { remaining := publication.VideoBudget() @@ -556,14 +553,12 @@ type publicationAudioContext struct { func (context publicationAudioContext) WriteStream() webrtc.TrackLocalWriter { return context.writer } -// Opus uses its one existing encoder at an explicit fixed ceiling. Reserve its -// bytes from allocator capacities. TWCC observes video; reserving unobserved -// audio is conservative and does not claim that the estimator acknowledged it. +// TWCC measures video and its probes. Audio shares the path but is not counted +// in this estimate; subtracting it here would understate the video budget. type publicationBWE struct { *sendsidebwe.SendSideBWE listener bwe.BWEListener target atomic.Int64 - audio atomic.Int64 observed atomic.Bool } @@ -582,14 +577,14 @@ func (bandwidth *publicationBWE) OnCongestionStateChange(from, to bwe.Congestion if to == bwe.CongestionStateCongested && capacity > 0 && capacity < bandwidth.target.Load() { bandwidth.target.Store(capacity) } - bandwidth.listener.OnCongestionStateChange(from, to, max(bandwidth.target.Load()-bandwidth.audio.Load(), 0)) + bandwidth.listener.OnCongestionStateChange(from, to, bandwidth.target.Load()) } func (bandwidth *publicationBWE) ProbeClusterFinalize() (ccutils.ProbeSignal, int64, bool) { signal, capacity, done := bandwidth.SendSideBWE.ProbeClusterFinalize() if done && signal == ccutils.ProbeSignalNotCongesting && capacity > bandwidth.target.Load() { bandwidth.target.Store(capacity) } - return signal, max(bandwidth.target.Load()-bandwidth.audio.Load(), 0), done + return signal, bandwidth.target.Load(), done } type publicationPacer struct { diff --git a/internal/media/forwarding/publication_test.go b/internal/media/forwarding/publication_test.go index bf5d36cd3..9dc2d8570 100644 --- a/internal/media/forwarding/publication_test.go +++ b/internal/media/forwarding/publication_test.go @@ -25,19 +25,17 @@ import ( func TestPublicationEmptyProbeCannotIncreaseCapacity(t *testing.T) { const target = int64(1_000_000) - const audio = int64(64_000) log := logger.GetDiscardLogger() bandwidth := &publicationBWE{SendSideBWE: sendsidebwe.NewSendSideBWE(sendsidebwe.SendSideBWEParams{ Config: sendsidebwe.DefaultSendSideBWEConfig, Logger: log, })} bandwidth.target.Store(target) - bandwidth.audio.Store(audio) queue := &publicationPacer{boundedPacer: newBoundedPacer( pacer.NewNoQueue(log, nil), log, 500, )} defer queue.Stop() info := ccutils.ProbeClusterInfo{Id: 1, CreatedAt: time.Now(), Goal: ccutils.ProbeClusterGoal{ - AvailableBandwidthBps: int(target), ExpectedUsageBps: int(target - audio), + AvailableBandwidthBps: int(target), ExpectedUsageBps: int(target), DesiredBps: int(target * 2), Duration: bandwidth.ProbeDuration(), }} info.Goal.DesiredBytes = int(float64(info.Goal.DesiredBps) * info.Goal.Duration.Seconds() / 8) @@ -53,7 +51,7 @@ func TestPublicationEmptyProbeCannotIncreaseCapacity(t *testing.T) { for { signal, capacity, done := bandwidth.ProbeClusterFinalize() if done { - if signal != ccutils.ProbeSignalInconclusive || capacity > target-audio || bandwidth.target.Load() != target { + if signal != ccutils.ProbeSignalInconclusive || capacity > target || bandwidth.target.Load() != target { t.Fatalf("empty probe raised publication capacity: signal=%v capacity=%d target=%d", signal, capacity, bandwidth.target.Load()) } break @@ -76,12 +74,10 @@ func TestPublicationProbeUsesAcknowledgedCapacity(t *testing.T) { func checkPublicationProbeCapacity(t *testing.T, received bool) { t.Helper() const target = int64(100_000) - const audio = int64(64_000) bandwidth := &publicationBWE{SendSideBWE: sendsidebwe.NewSendSideBWE(sendsidebwe.SendSideBWEParams{ Config: sendsidebwe.DefaultSendSideBWEConfig, Logger: logger.GetDiscardLogger(), })} bandwidth.target.Store(target) - bandwidth.audio.Store(audio) bandwidth.SetBWEListener(&publicationCapacityRecorder{}) // A real non-congesting probe after reset still carries the library's // initial capacity. A successful 800 kbps probe cannot prove 100 Mbps. @@ -123,11 +119,11 @@ func checkPublicationProbeCapacity(t *testing.T, received bool) { if received { // Allow the first-packet/time-window difference in the library's // acknowledged rate, but never its untouched 100 Mbps initial value. - if signal != ccutils.ProbeSignalNotCongesting || capacity <= target-audio || capacity > 1_000_000-audio { - t.Fatalf("probe promoted unmeasured capacity: signal=%v committed=%d sent=%.0f", signal, capacity+audio, info.Result.Bitrate()) + if signal != ccutils.ProbeSignalNotCongesting || capacity <= target || capacity > 1_000_000 { + t.Fatalf("unexpected acknowledged probe capacity: signal=%v committed=%d sent=%.0f", signal, capacity, info.Result.Bitrate()) } - } else if signal != ccutils.ProbeSignalInconclusive || capacity != target-audio { - t.Fatalf("unacknowledged probe changed capacity: signal=%v committed=%d", signal, capacity+audio) + } else if signal != ccutils.ProbeSignalInconclusive || capacity != target { + t.Fatalf("unacknowledged probe changed capacity: signal=%v committed=%d", signal, capacity) } return } @@ -148,16 +144,15 @@ func TestPublicationCongestionUsesCommittedCapacity(t *testing.T) { recorder := &publicationCapacityRecorder{} bandwidth := &publicationBWE{listener: recorder} bandwidth.target.Store(1_200_000) - bandwidth.audio.Store(64_000) bandwidth.OnCongestionStateChange(bwe.CongestionStateNone, bwe.CongestionStateEarlyWarning, 100_000_000) bandwidth.OnCongestionStateChange(bwe.CongestionStateEarlyWarning, bwe.CongestionStateCongested, 100_000_000) - if bandwidth.target.Load() != 1_200_000 || recorder.capacity != 1_136_000 { + if bandwidth.target.Load() != 1_200_000 || recorder.capacity != 1_200_000 { t.Fatalf("publication promoted an unverified capacity: target=%d allocation=%d", bandwidth.target.Load(), recorder.capacity) } bandwidth.OnCongestionStateChange(bwe.CongestionStateCongested, bwe.CongestionStateCongested, 500_000) bandwidth.OnCongestionStateChange(bwe.CongestionStateCongested, bwe.CongestionStateNone, 800_000) - if bandwidth.target.Load() != 500_000 || recorder.capacity != 436_000 { - t.Fatalf("publication disagrees with its committed audio/video budget: target=%d allocation=%d", bandwidth.target.Load(), recorder.capacity) + if bandwidth.target.Load() != 500_000 || recorder.capacity != 500_000 { + t.Fatalf("publication disagrees with its committed video budget: target=%d allocation=%d", bandwidth.target.Load(), recorder.capacity) } } @@ -187,9 +182,8 @@ func TestPublicationSimulcastReuseAudioDemandAndClose(t *testing.T) { publication, err := NewPublication(TransportOptions{Source: source.Source, Settings: settings, ConnectionID: "publication", InitialBitrate: 1_000_000, Audio: audio}) check(err) defer publication.Close() - publication.SetAudioBitrate(64_000) - if publication.VideoBudget() != 936_000 { - t.Fatal("audio did not reserve the shared budget") + if publication.VideoBudget() != 1_000_000 { + t.Fatal("audio changed the video capacity estimate") } if len(publication.PC.GetSenders()) != 2 || len(publication.PC.GetSenders()[0].GetParameters().Encodings) != 2 { t.Fatal("publication must use one simulcast video sender and one audio sender") @@ -293,7 +287,7 @@ func TestPublicationSimulcastReuseAudioDemandAndClose(t *testing.T) { t.Fatalf("successful RTP writes were not accounted per encoding: %+v", counters) } // Once real bitrate measurements exist, one shared capacity update must - // constrain the sum of both RID allocations after reserving Opus. + // constrain the sum of both video RID allocations. for { _, rates := source.GetLayeredBitrate() if rates[0][0] > 0 && rates[1][0] > 0 { @@ -306,12 +300,12 @@ func TestPublicationSimulcastReuseAudioDemandAndClose(t *testing.T) { t.Fatal("publication did not measure both available encodings") } } - publication.bandwidth.OnCongestionStateChange(bwe.CongestionStateNone, bwe.CongestionStateCongested, 65_000) + publication.bandwidth.OnCongestionStateChange(bwe.CongestionStateNone, bwe.CongestionStateCongested, 1_000) for publication.tracks[0].BandwidthRequested()+publication.tracks[1].BandwidthRequested() > 1_000 { select { case <-time.After(5 * time.Millisecond): case <-ctx.Done(): - t.Fatal("simulcast allocations exceeded their shared post-audio capacity") + t.Fatal("simulcast allocations exceeded their shared video capacity") } } if publication.LowestLayerBudget() != 1_000 { @@ -323,7 +317,7 @@ func TestPublicationSimulcastReuseAudioDemandAndClose(t *testing.T) { // Restore this allocation fixture's capacity explicitly. A congestion event // cannot prove an increase; this test covers RID reuse, not bandwidth probing. publication.bandwidth.target.Store(1_000_000) - publication.allocator.SetChannelCapacity(936_000) + publication.allocator.SetChannelCapacity(1_000_000) check(publication.SetActiveCount(1)) if publication.RequiredActiveCount() != 1 { t.Fatal("publication did not reduce its direct demand") diff --git a/internal/media/forwarding/recovery_test.go b/internal/media/forwarding/recovery_test.go index ce003ac33..6edcae045 100644 --- a/internal/media/forwarding/recovery_test.go +++ b/internal/media/forwarding/recovery_test.go @@ -2,11 +2,16 @@ package forwarding import ( "context" + "encoding/binary" + "math" + "math/rand/v2" "os" + "sync" "sync/atomic" "testing" "time" + "github.com/TNTcraftHIM/Piik/internal/app/nativeaudio" "github.com/TNTcraftHIM/Piik/internal/media/encoded" "github.com/livekit/livekit-server/pkg/sfu" "github.com/pion/ice/v4" @@ -14,15 +19,24 @@ import ( "github.com/pion/logging" "github.com/pion/transport/v4/vnet" "github.com/pion/webrtc/v4" + rtmedia "github.com/pion/webrtc/v4/pkg/media" + "github.com/thesyncim/gopus" ) // Explicit CPU-only network gate: synthetic VP8-shaped payload, actual Pion -// ICE/DTLS/RTP/TWCC, and the production allocator/pacer/probe driver. No decoder, +// ICE/DTLS/RTP/TWCC, and the production allocator/pacer/probe driver. The audio +// case adds real Opus encode/decode and live bitrate changes. No video decoder, // capture device or physical network participates. func TestTransportConstrainedDormantRecovery(t *testing.T) { if os.Getenv("PIIK_TRANSPORT_RECOVERY") == "" { t.Skip("set PIIK_TRANSPORT_RECOVERY=1 for the bounded virtual-network gate") } + t.Run("video-only", func(t *testing.T) { checkTransportRecovery(t, false) }) + t.Run("numeric-audio", func(t *testing.T) { checkTransportRecovery(t, true) }) +} + +func checkTransportRecovery(t *testing.T, withAudio bool) { + t.Helper() ctx, cancel := context.WithTimeout(t.Context(), 32*time.Second) defer cancel() check := func(err error) { @@ -58,8 +72,36 @@ func TestTransportConstrainedDormantRecovery(t *testing.T) { sendSettings.SetNet(sendNet) sendSettings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) sendSettings.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled) + var audio *webrtc.TrackLocalStaticSample + var encoder *nativeaudio.Encoder + var audioTicks <-chan time.Time + var pcm []byte + if withAudio { + audio, err = webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, + SDPFmtpLine: "minptime=10;useinbandfec=1", + }, "audio", "recovery") + check(err) + encoder, err = nativeaudio.NewEncoder(320_000) + check(err) + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + audioTicks = ticker.C + pcm = make([]byte, nativeaudio.FrameBytes) + random := rand.New(rand.NewPCG(42, 1)) + for frame := range nativeaudio.FrameSamples { + for channel := range nativeaudio.Channels { + value := 8000*math.Sin(2*math.Pi*float64(500+channel*1000)*float64(frame)/48000) + 4000*(random.Float64()*2-1) + binary.LittleEndian.PutUint16(pcm[(frame*2+channel)*2:], uint16(int16(value))) + } + } + } + var audioTrack webrtc.TrackLocal + if audio != nil { + audioTrack = audio + } transport, err := NewTransport(TransportOptions{Source: source.Source, Settings: sendSettings, - ConnectionID: "recovery-child", InitialBitrate: 1_200_000}) + ConnectionID: "recovery-child", InitialBitrate: 1_200_000, Audio: audioTrack}) check(err) defer transport.Close() receiveSettings := webrtc.SettingEngine{} @@ -73,9 +115,39 @@ func TestTransportConstrainedDormantRecovery(t *testing.T) { receiver, err := webrtc.NewAPI(webrtc.WithMediaEngine(media), webrtc.WithInterceptorRegistry(registry), webrtc.WithSettingEngine(receiveSettings)).NewPeerConnection(webrtc.Configuration{}) check(err) + var readers sync.WaitGroup + defer readers.Wait() defer receiver.Close() - var received atomic.Uint64 + var received, audioDecoded, audioErrors atomic.Uint64 receiver.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + readers.Add(1) + defer readers.Done() + if track.Kind() == webrtc.RTPCodecTypeAudio { + decoder, err := gopus.NewDecoder(gopus.DefaultDecoderConfig(48000, 2)) + if err != nil { + audioErrors.Add(1) + return + } + pcm := make([]int16, nativeaudio.FrameSamples*nativeaudio.Channels) + for { + packet, _, err := track.ReadRTP() + if err != nil { + return + } + count, err := decoder.DecodeInt16(packet.Payload, pcm) + if err != nil || count != nativeaudio.FrameSamples { + audioErrors.Add(1) + continue + } + var energy int64 + for _, sample := range pcm { + energy += int64(sample) * int64(sample) + } + if energy > int64(len(pcm))*100 { + audioDecoded.Add(1) + } + } + } for { packet, _, readErr := track.ReadRTP() if readErr != nil { @@ -116,12 +188,26 @@ func TestTransportConstrainedDormantRecovery(t *testing.T) { ticker := time.NewTicker(period) defer ticker.Stop() start, lastReport := time.Now(), time.Now() - var previousReceived uint64 - var dormant, recovered bool + var previousReceived, previousAudio uint64 + var dormant bool + var recoveredWindows, audioWindows int minimum := int64(1_200_000) - for frame := 0; ; frame++ { + frame := 0 + for { select { case <-ticker.C: + case <-audioTicks: + bitrate := 320_000 + if elapsed := time.Since(start); elapsed >= 8*time.Second && elapsed < 16*time.Second { + bitrate = 96_000 + } + if encoder.Bitrate() != bitrate { + check(encoder.SetBitrate(bitrate)) + } + packet, err := encoder.Encode(pcm) + check(err) + check(audio.WriteSample(rtmedia.Sample{Data: packet, Duration: 20 * time.Millisecond})) + continue case <-ctx.Done(): t.Fatal(ctx.Err()) } @@ -156,6 +242,7 @@ func TestTransportConstrainedDormantRecovery(t *testing.T) { copy(data, sfu.VP8KeyFrame8x8) check(source.WriteFrame(layer, encoded.Frame{Data: data, PTS: pts, Duration: period})) } + frame++ if now.Sub(lastReport) >= time.Second { bytes := received.Load() receivedBps := float64((bytes-previousReceived)*8) / now.Sub(lastReport).Seconds() @@ -164,15 +251,23 @@ func TestTransportConstrainedDormantRecovery(t *testing.T) { elapsed.Seconds(), capacity, state.VideoBudget, state.Target, state.Current, high, receivedBps, transport.estimator.CongestionState(), transport.paddingReady, transport.probe.Id, transport.Egress()) transport.mu.Unlock() - if elapsed > 15*time.Second && state.Current == 1 && receivedBps > 800_000 { - recovered = true + if elapsed >= 20*time.Second && state.Current == 1 && receivedBps > 800_000 { + recoveredWindows++ + } + decoded := audioDecoded.Load() + if withAudio { + t.Logf("audio decoded=%d errors=%d bitrate=%d", decoded-previousAudio, audioErrors.Load(), encoder.Bitrate()) + if elapsed >= 20*time.Second && decoded-previousAudio >= 35 { + audioWindows++ + } } + previousAudio = decoded lastReport, previousReceived = now, bytes } } - if !dormant || minimum >= 1_200_000 || !recovered { - t.Fatalf("missing real constrained/dormant/released recovery: dormant=%v minimum=%d recovered=%v state=%+v", - dormant, minimum, recovered, transport.Output.State()) + if !dormant || minimum >= 1_200_000 || recoveredWindows < 5 || withAudio && (audioWindows < 5 || audioErrors.Load() != 0) { + t.Fatalf("missing constrained/dormant/released recovery: dormant=%v minimum=%d videoWindows=%d audioWindows=%d audioErrors=%d state=%+v", + dormant, minimum, recoveredWindows, audioWindows, audioErrors.Load(), transport.Output.State()) } check(transport.Close()) if transport.prober.IsRunning() { diff --git a/internal/media/forwarding/transport.go b/internal/media/forwarding/transport.go index 32ac1521f..eb33fed90 100644 --- a/internal/media/forwarding/transport.go +++ b/internal/media/forwarding/transport.go @@ -55,7 +55,6 @@ type Transport struct { mu sync.Mutex attached bool closed bool - audioBitrate uint32 paddingReady bool closeOnce sync.Once closeErr error @@ -274,6 +273,7 @@ func validateReplacement(current, next *Source) error { return nil } +// TargetBitrate reports capacity for TWCC-tracked video, not total link capacity. func (transport *Transport) TargetBitrate() (int, bool) { return int(transport.target.Load()), transport.observed.Load() } @@ -299,20 +299,8 @@ func (transport *Transport) Egress() EgressStats { return transport.pacer.stats() } -// SetAudioBitrate reserves the current audio owner's contribution to this PC's -// shared estimate. Zero means no audio demand; the transport invents no profile. -func (transport *Transport) SetAudioBitrate(bitrate uint32) { - transport.mu.Lock() - defer transport.mu.Unlock() - if !transport.closed { - transport.audioBitrate = bitrate - transport.refreshBudgetLocked() - } -} - func (transport *Transport) refreshBudgetLocked() { - remaining := transport.target.Load() - int64(transport.audioBitrate) - transport.Output.SetBudget(max(remaining, 0)) + transport.Output.SetBudget(transport.target.Load()) } func (transport *Transport) Close() error { @@ -446,7 +434,6 @@ func (transport *Transport) driveProbeLocked() { if delta <= 0 { return } - usage += int64(transport.audioBitrate) goal := ccutils.ProbeClusterGoal{AvailableBandwidthBps: int(transport.target.Load()), ExpectedUsageBps: int(usage), DesiredBps: int(usage + max(delta*config.ProbeOveragePct/100, config.ProbeMinBps)), Duration: transport.estimator.ProbeDuration()} diff --git a/internal/media/forwarding/transport_test.go b/internal/media/forwarding/transport_test.go index 3a77fce4b..a0d5ef4fe 100644 --- a/internal/media/forwarding/transport_test.go +++ b/internal/media/forwarding/transport_test.go @@ -70,9 +70,8 @@ func TestTransportVideoFeedbackAudioAndClose(t *testing.T) { check(err) defer transport.Close() transport.Output.SetMaxSpatialLayer(0) - transport.SetAudioBitrate(64_000) - if state := transport.Output.State(); state.VideoBudget != 936_000 { - t.Fatalf("audio reservation did not reach the video codec budget: %+v", state) + if state := transport.Output.State(); state.VideoBudget != 1_000_000 { + t.Fatalf("audio changed the video capacity estimate: %+v", state) } if len(source.GetDownTracks()) != 0 { t.Fatal("unconnected transport claimed source output") diff --git a/internal/server/sfu/media.go b/internal/server/sfu/media.go index ad90a3efd..4c2be04f7 100644 --- a/internal/server/sfu/media.go +++ b/internal/server/sfu/media.go @@ -392,15 +392,6 @@ func (media *Media) UpdatePublication(fence ResourceFence, connectionID string, metadata.Formats, metadata.RIDs = slices.Clone(metadata.Formats), slices.Clone(metadata.RIDs) publication.metadata = metadata publication.wakeDemand() - bitrate := uint32(0) - if metadata.Audio { - bitrate = metadata.AudioBitrate - } - for _, subscriber := range publication.subscribers { - if subscriber.transport != nil { - subscriber.transport.SetAudioBitrate(bitrate) - } - } return nil } @@ -579,9 +570,6 @@ func (media *Media) PrepareSubscriber(ctx context.Context, fence SubscriptionFen if err != nil { return webrtc.SessionDescription{}, err } - if metadata.Audio { - transport.SetAudioBitrate(metadata.AudioBitrate) - } publication.mu.Lock() if publication.closed || publication.failure != nil || publication.subscribers[key] != subscriber || ctx.Err() != nil || lifetime.Err() != nil { publication.mu.Unlock() From d9e477d091f954198a5879ba7ec9f698b405fec4 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:26:33 +0800 Subject: [PATCH 088/262] test(media): verify receiver admission preserves healthy siblings --- docs/research/browser-screen-audio-quality.md | 6 ++- internal/app/mediaedge/edge_test.go | 37 ++++++++++++++++++- internal/server/sfu/media_test.go | 27 +++++++++++++- 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/docs/research/browser-screen-audio-quality.md b/docs/research/browser-screen-audio-quality.md index b45c08c7d..0591bdea5 100644 --- a/docs/research/browser-screen-audio-quality.md +++ b/docs/research/browser-screen-audio-quality.md @@ -105,7 +105,11 @@ retirement also passed; the Browser run measured stereo separation. An ordinary Browser relay forwarded received tracks to two children at 320 kbps, with advancing decoded audio and successful cleanup. A direct Browser control now keeps requested 256/320 kbps below an explicit 192 kbps receive limit, while a -510 kbps receiver permits both rates. A native real-Opus control through +510 kbps receiver permits both rates. Native and SFU adapter controls reject +an insufficient 192 kbps answer before binding it; retiring or correcting that +preparation leaves the original source and healthy sibling delivering media. +These controls do not establish the complete room-route response to rejection. +A native real-Opus control through 6 Mbps -> 500 kbps -> 6 Mbps shaping retains decoded audio during live 320/96/320 kbps changes, with sustained high video after the shared [probe measurement and video-budget repairs](./node-local-media-design.md#send-queue-recovery). diff --git a/internal/app/mediaedge/edge_test.go b/internal/app/mediaedge/edge_test.go index cc2424052..95a4269b9 100644 --- a/internal/app/mediaedge/edge_test.go +++ b/internal/app/mediaedge/edge_test.go @@ -696,11 +696,11 @@ func TestAudioSourceDeliversOpusOnTheVideoPeerConnection(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - video, err := engine.NewSource("h264", 1, 1, nil) + video, err := engine.NewSource("h264", 2, 1, nil) if err != nil { t.Fatal(err) } - audio, err := engine.NewAudioSource(1, 128_000) + audio, err := engine.NewAudioSource(2, 128_000) if err != nil { t.Fatal(err) } @@ -722,6 +722,39 @@ func TestAudioSourceDeliversOpusOnTheVideoPeerConnection(t *testing.T) { } }) connectEdgeToReceiver(t, edge, receiver) + limited, err := engine.NewEdge(video, EdgeOptions{ConnectionID: "limited-audio-edge", Audio: audio}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = limited.Close() }) + limitedReceiver := newAudioReceiver(t) + t.Cleanup(func() { _ = limitedReceiver.Close() }) + offer, err := limited.CreateOffer() + if err != nil { + t.Fatal(err) + } + if err = limitedReceiver.SetRemoteDescription(offer); err != nil { + t.Fatal(err) + } + answer, err := (&Receiver{connection: limitedReceiver}).createAnswer() + if err != nil { + t.Fatal(err) + } + answer.SDP = strings.ReplaceAll(answer.SDP, "maxaveragebitrate=510000", "maxaveragebitrate=192000") + if err = limited.SetAnswer(answer); err == nil { + t.Fatal("shared audio accepted an insufficient receiver limit") + } + if limited.connection.RemoteDescription() != nil { + t.Fatal("rejected answer was applied before admission") + } + if err = limited.Close(); err != nil { + t.Fatal(err) + } + // Rejecting and retiring this edge must leave the shared audio source and + // the first receiver usable, without reducing the source's live setting. + if audio.configuredBitrate() != 128_000 { + t.Fatal("receiver admission changed the shared audio setting") + } if err = audio.WritePCM(make([]byte, nativeaudio.FrameBytes), 20*time.Millisecond); err != nil { t.Fatal(err) } diff --git a/internal/server/sfu/media_test.go b/internal/server/sfu/media_test.go index 3fe701873..19b78aa78 100644 --- a/internal/server/sfu/media_test.go +++ b/internal/server/sfu/media_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net" + "strings" "sync" "sync/atomic" "testing" @@ -310,7 +311,31 @@ func runMediaFanout(t *testing.T, codec string, layerCount int) { } answer, answerErr := receiver.CreateAnswer(nil) check(answerErr) - check(service.ApplySubscriberAnswer(subscription, "subscriber_123456", waitGathering(receiver, answer))) + answer = waitGathering(receiver, answer) + if index == 1 { + // Admission of one limited receiver must not bind its incompatible + // audio or disturb the publication and its already-playing sibling. + limited := answer + limited.SDP = strings.ReplaceAll(limited.SDP, "maxaveragebitrate=510000", "maxaveragebitrate=192000") + if limited.SDP == answer.SDP { + t.Fatal("receiver fixture did not declare its Opus limit") + } + if service.ApplySubscriberAnswer(subscription, "subscriber_123456", limited) == nil { + t.Fatal("shared audio accepted an insufficient receiver limit") + } + if physical.transport.PC.RemoteDescription() != nil { + t.Fatal("rejected answer was applied before admission") + } + beforeVideo, beforeAudio := videoPackets[0].Load(), audioPackets[0].Load() + for videoPackets[0].Load() < beforeVideo+3 || audioPackets[0].Load() < beforeAudio+3 { + select { + case <-time.After(10 * time.Millisecond): + case <-ctx.Done(): + t.Fatal("receiver rejection interrupted its healthy sibling") + } + } + } + check(service.ApplySubscriberAnswer(subscription, "subscriber_123456", answer)) } for videoPackets[0].Load() < 3 || videoPackets[1].Load() < 3 || audioPackets[0].Load() < 3 || audioPackets[1].Load() < 3 { From 08a99f7479b55bb3a551f6be8753e61ee0e03f2e Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 02:59:08 +0800 Subject: [PATCH 089/262] docs(research): record complete App address-change limits --- docs/research/nat-traversal.md | 21 ++++++++++++++++++--- docs/todo.md | 11 ++++++----- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index a0b117086..aea279b16 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -691,9 +691,24 @@ current wildcard bind and a non-loopback selected pair. Mozilla also keeps These controls establish a compatibility limit, not its complete Browser cause or public-Site permission behavior. Keep the current address policy. The Host currently ends sharing when its required local bridge fails; the Viewer delegates -that failure to its existing bounded connection-replacement owner. Complete App -handoff acceptance must cover both, without treating the remote tunnel's survival -as proof that local playback survived. +that failure to its existing bounded connection-replacement owner. + +Complete App UI controls on 2026-10-10 confirm this distinction in a disposable +Linux network namespace, with automatic topology optimization disabled and no +product source changes. Chromium 152 retains its loopback playback connection +when the namespace's LAN address changes. Firefox 151 selects that LAN address: +the Viewer resumes decoded pictures through the existing Browser fallback after +the local connection fails, roughly 30 seconds later. A Firefox Host instead +ends sharing on that preview failure, including a still-delivering downstream +connection; the Chromium Host control continues. The Host source is generated +VP8 delivered through the real capture-process interface, not physical capture. +Acceptance checks fresh rendered frames at the end, not accumulated frames from +before the interruption. All controls retire their App, Browsers and interfaces. +These are ordinary-transport controls of the current App, not integrated +Magicsock or physical roaming evidence. A faster remote tunnel alone cannot +establish complete App handoff; local playback composition remains an adoption +boundary. Retain the existing failure owners and address policy until a bounded +alternative passes both Browser and Host/Viewer controls. Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof diff --git a/docs/todo.md b/docs/todo.md index f23a4bae3..617e59f8d 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -1,6 +1,6 @@ # Current TODO Ledger -Last reviewed: 2026-10-09 +Last reviewed: 2026-10-10 The current 2.0 preparation includes independent investigation across this ledger. Explicit owner holds remain in force; an investigation does not itself @@ -75,10 +75,11 @@ Implementation needs a concrete useful result within the product model. pipeline and physical networks; it does not establish another reachable NAT class. Live-encoder/decoded-picture controls now separate shared forwarding queue delay from the local playback leg losing its selected LAN address. The shared - send-queue repair passes constrained-link recovery; verify the App's local - Browser bridge during address changes. A loopback-only policy fails the Firefox - control; retain the current address policy and verify both Host and Viewer - lifecycle behavior before claiming complete App recovery. + send-queue repair passes constrained-link recovery. The complete App controls + confirm that its local Browser bridge is a separate handoff limit, especially + for Host preview. Resolve that composition within the existing owners before + claiming complete App recovery. A loopback-only policy fails the Firefox + control; retain the current address policy. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) and RTCP/MTU boundaries, retained-flow recovery beyond the passing serial From 75fb78ced9f47ce009259735da6efdfe54612f78 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:29:15 +0800 Subject: [PATCH 090/262] fix(media): refresh interface addresses during ICE gathering --- docs/research/nat-traversal.md | 16 +++ docs/todo.md | 5 +- internal/app/mediaedge/ice_socket.go | 3 +- internal/media/forwarding/ice.go | 29 +++++ internal/media/forwarding/ice_linux_test.go | 120 ++++++++++++++++++++ internal/server/app/app.go | 3 +- 6 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 internal/media/forwarding/ice.go create mode 100644 internal/media/forwarding/ice_linux_test.go diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index aea279b16..8cd7d204c 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -710,6 +710,22 @@ establish complete App handoff; local playback composition remains an adoption boundary. Retain the existing failure owners and address policy until a bounded alternative passes both Browser and Host/Viewer controls. +The local restart control exposed a separate address-cache defect: Pion ICE +v4.4.0's [wildcard UDP mux](https://github.com/pion/ice/blob/v4.4.0/udp_mux.go) +retains its own `stdnet.Net` interface snapshot across +gatherings. After an interface address changes, both ICE restart and a new SFU +connection sharing that mux can advertise the removed address. The shared mux +adapter now takes a private current snapshot when gathering; it changes neither +socket identity nor Pion's candidate filters and adds no monitor or timer. +The isolated regression covers restart and a new connection; restoring the +former constructor fails on the stale candidate. Firefox also resumes fresh +rendered frames on the same track after a standard ICE restart, including from +the failed state, while a separately bound loopback sibling keeps receiving. +That browser control uses complete SDP through a lab-only restart entry point. +It establishes the underlying repair, not product local-bridge recovery: the +App's current local bridge still negotiates only once, and its existing failure +owners remain unchanged. + Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof required fresh bidirectional delivery after DERP closed. One side reporting a diff --git a/docs/todo.md b/docs/todo.md index 617e59f8d..3df84315e 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -79,7 +79,10 @@ Implementation needs a concrete useful result within the product model. confirm that its local Browser bridge is a separate handoff limit, especially for Host preview. Resolve that composition within the existing owners before claiming complete App recovery. A loopback-only policy fails the Firefox - control; retain the current address policy. + control; retain the current address policy. Fresh interface gathering fixes + stale host candidates, and a lab-only standard ICE restart preserves the local + track. Product bridge renegotiation, its generation fence and bounded recovery + are still unimplemented; do not count that experiment as complete App recovery. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) and RTCP/MTU boundaries, retained-flow recovery beyond the passing serial diff --git a/internal/app/mediaedge/ice_socket.go b/internal/app/mediaedge/ice_socket.go index c3e2bac4a..0e458aee9 100644 --- a/internal/app/mediaedge/ice_socket.go +++ b/internal/app/mediaedge/ice_socket.go @@ -12,6 +12,7 @@ import ( "github.com/TNTcraftHIM/Piik/internal/app/portmapping" "github.com/TNTcraftHIM/Piik/internal/diagnostics" + "github.com/TNTcraftHIM/Piik/internal/media/forwarding" "github.com/pion/ice/v4" "github.com/pion/stun/v3" "github.com/pion/webrtc/v4" @@ -45,7 +46,7 @@ func (engine *Engine) newICESocket(servers []webrtc.ICEServer, mapping bool) (*i socket := &iceSocket{ engine: engine, ctx: ctx, cancel: cancel, localPort: connection.LocalAddr().(*net.UDPAddr).Port, - mux: ice.NewUDPMuxDefault(ice.UDPMuxParams{ + mux: forwarding.NewUDPMux(ice.UDPMuxParams{ Logger: diagnostics.PionLoggerFactory().NewLogger("piik-ice"), UDPConn: connection, }), } diff --git a/internal/media/forwarding/ice.go b/internal/media/forwarding/ice.go new file mode 100644 index 000000000..cafec6f81 --- /dev/null +++ b/internal/media/forwarding/ice.go @@ -0,0 +1,29 @@ +package forwarding + +import ( + "github.com/pion/ice/v4" + "github.com/pion/transport/v4" + "github.com/pion/transport/v4/stdnet" +) + +// NewUDPMux retains Pion's socket and candidate policy, but reads current local +// addresses at each gathering. Its default stdnet snapshot otherwise survives +// ICE restart (and every new SFU connection sharing that mux). +func NewUDPMux(params ice.UDPMuxParams) *ice.UDPMuxDefault { + if params.Net == nil { + params.Net = &muxNetwork{} + } + return ice.NewUDPMuxDefault(params) +} + +// UDPMux uses Net only to enumerate interfaces. Take a fresh, private snapshot +// per read instead of mutating a shared stdnet.Net during concurrent gatherings. +type muxNetwork struct{ stdnet.Net } + +func (*muxNetwork) Interfaces() ([]*transport.Interface, error) { + network, err := stdnet.NewNet() + if err != nil { + return nil, err + } + return network.Interfaces() +} diff --git a/internal/media/forwarding/ice_linux_test.go b/internal/media/forwarding/ice_linux_test.go new file mode 100644 index 000000000..59828e25d --- /dev/null +++ b/internal/media/forwarding/ice_linux_test.go @@ -0,0 +1,120 @@ +package forwarding + +import ( + "net" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/pion/ice/v4" + "github.com/pion/webrtc/v4" +) + +// Run only in a disposable network namespace with lab0 at 10.77.0.2/24. +// This exercises real interface replacement without changing the host network. +func TestUDPMuxRefreshesHostCandidates(t *testing.T) { + if os.Getenv("PIIK_INTERFACE_LAB") != "1" { + t.Skip("requires the isolated Linux interface fixture") + } + self, err := os.Readlink("/proc/self/ns/net") + if err != nil { + t.Fatal(err) + } + initial, err := os.Readlink("/proc/1/ns/net") + if err != nil || self == initial { + t.Fatal("refusing to change the initial network namespace") + } + iface, err := net.InterfaceByName("lab0") + if err != nil { + t.Fatal(err) + } + addresses, err := iface.Addrs() + if err != nil { + t.Fatal(err) + } + found := false + for _, address := range addresses { + found = found || address.String() == "10.77.0.2/24" + } + if !found { + t.Fatal("fixture address is absent") + } + change := func(action, address string) { + t.Helper() + if output, err := exec.Command("ip", "addr", action, address, "dev", "lab0").CombinedOutput(); err != nil { + t.Fatalf("fixture address change: %v: %s", err, output) + } + } + udp, err := net.ListenPacket("udp4", "0.0.0.0:0") + if err != nil { + t.Fatal(err) + } + mux := NewUDPMux(ice.UDPMuxParams{UDPConn: udp}) + t.Cleanup(func() { _ = mux.Close() }) + settings := webrtc.SettingEngine{} + settings.SetICEUDPMux(mux) + settings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + newPeer := func() (*webrtc.PeerConnection, *webrtc.PeerConnection) { + t.Helper() + peer, err := webrtc.NewAPI(webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = peer.Close() }) + if _, err = peer.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo); err != nil { + t.Fatal(err) + } + remote, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = remote.Close() }) + return peer, remote + } + gather := func(peer, remote *webrtc.PeerConnection, restart bool, want, reject string) { + t.Helper() + offer, err := peer.CreateOffer(&webrtc.OfferOptions{ICERestart: restart}) + if err != nil { + t.Fatal(err) + } + complete := webrtc.GatheringCompletePromise(peer) + if err = peer.SetLocalDescription(offer); err != nil { + t.Fatal(err) + } + select { + case <-complete: + case <-time.After(3 * time.Second): + t.Fatal("candidate gathering did not finish") + } + sdp := peer.LocalDescription().SDP + if !strings.Contains(sdp, " "+want+" ") || (reject != "" && strings.Contains(sdp, " "+reject+" ")) { + t.Fatalf("gathering did not replace the old interface address: want=%s reject=%s", want, reject) + } + if err = remote.SetRemoteDescription(*peer.LocalDescription()); err != nil { + t.Fatal(err) + } + answer, err := remote.CreateAnswer(nil) + if err != nil { + t.Fatal(err) + } + if err = remote.SetLocalDescription(answer); err != nil { + t.Fatal(err) + } + if err = peer.SetRemoteDescription(answer); err != nil { + t.Fatal(err) + } + } + peer, remote := newPeer() + gather(peer, remote, false, "10.77.0.2", "") + change("del", "10.77.0.2/24") + t.Cleanup(func() { change("add", "10.77.0.2/24") }) + change("add", "10.77.0.3/24") + t.Cleanup(func() { change("del", "10.77.0.3/24") }) + // App recovery retains its connection and socket; SFU also reuses its mux + // across independent connections. Both must enumerate the current address. + gather(peer, remote, true, "10.77.0.3", "10.77.0.2") + peer, remote = newPeer() + gather(peer, remote, false, "10.77.0.3", "10.77.0.2") +} diff --git a/internal/server/app/app.go b/internal/server/app/app.go index 4f3e75b47..7a7ae1d4c 100644 --- a/internal/server/app/app.go +++ b/internal/server/app/app.go @@ -30,6 +30,7 @@ import ( "time" "github.com/TNTcraftHIM/Piik/internal/diagnostics" + "github.com/TNTcraftHIM/Piik/internal/media/forwarding" "github.com/TNTcraftHIM/Piik/internal/server/config" "github.com/TNTcraftHIM/Piik/internal/server/room" "github.com/TNTcraftHIM/Piik/internal/server/sfu" @@ -250,7 +251,7 @@ func (s *Server) start(ctx context.Context) (int, error) { return 0, fmt.Errorf("Piik SFU listener failed: %w", listenErr) } factory := diagnostics.PionLoggerFactory() - s.mediaMux = ice.NewUDPMuxDefault(ice.UDPMuxParams{UDPConn: connection, Logger: factory.NewLogger("sfu-udp")}) + s.mediaMux = forwarding.NewUDPMux(ice.UDPMuxParams{UDPConn: connection, Logger: factory.NewLogger("sfu-udp")}) settings := webrtc.SettingEngine{LoggerFactory: factory} settings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) settings.SetICEUDPMux(s.mediaMux) From 132c73337164788c973c6e278fd4f8d99c96dc81 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 04:03:07 +0800 Subject: [PATCH 091/262] fix(media): recover local native bridges on retained connections Use bounded, offer-scoped ICE restart for the App's local Browser leg while preserving its stream and terminal owners. Gather local candidates in the SDP transaction and reject stale answers. Start legacy audio from the fixed output owner instead of racing source replacement. Validated with focused Browser/Go tests, complete Chromium/Firefox Host and Viewer address-change controls, and Linux race checks. Capture subprocess fixtures use GORACE=atexit_sleep_ms=0; restoring the old audio-owner read still reproduces the race under that setting. --- docs/research/nat-traversal.md | 43 ++- docs/standards/presentation-lifecycle.md | 8 +- docs/todo.md | 12 +- internal/app/mediaedge/edge.go | 78 +++++- internal/app/mediaedge/edge_test.go | 6 +- internal/app/mediaedge/local_edge_test.go | 94 +++++++ internal/app/mediaedge/publication.go | 2 +- .../app/mediaedge/receiver_overlap_test.go | 2 +- .../app/mediaedge/relay_chain_fixture_test.go | 2 +- internal/app/nativecontrol/session.go | 9 +- internal/app/nativecontrol/wire.go | 2 + internal/app/nativehost/audio_mix_test.go | 4 +- internal/app/nativehost/audio_test.go | 4 +- internal/app/nativehost/session.go | 32 ++- internal/app/nativeviewer/session.go | 31 ++- internal/app/nativeviewer/session_test.go | 2 +- src/client/native/client.ts | 9 +- src/client/native/media-bridge.ts | 254 +++++++++--------- tests/native-client.test.ts | 41 +++ tests/native-media-bridge.test.ts | 209 ++++++-------- 20 files changed, 530 insertions(+), 314 deletions(-) create mode 100644 internal/app/mediaedge/local_edge_test.go diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 8cd7d204c..c3f578dca 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -689,12 +689,13 @@ Firefox 155 fails that connection; the same Firefox fixture decodes with the current wildcard bind and a non-loopback selected pair. Mozilla also keeps [ICE loopback disabled by default](https://searchfox.org/firefox-main/source/modules/libpref/init/all.js). These controls establish a compatibility limit, not its complete Browser cause -or public-Site permission behavior. Keep the current address policy. The Host -currently ends sharing when its required local bridge fails; the Viewer delegates -that failure to its existing bounded connection-replacement owner. +or public-Site permission behavior. Keep the current address policy. A terminal +local-bridge failure ends the Host share or delegates Viewer recovery to its +existing connection owner. -Complete App UI controls on 2026-10-10 confirm this distinction in a disposable -Linux network namespace, with automatic topology optimization disabled and no +Before local-bridge recovery, complete App UI controls on 2026-10-10 confirmed +this distinction in a disposable Linux network namespace, with automatic +topology optimization disabled and no product source changes. Chromium 152 retains its loopback playback connection when the namespace's LAN address changes. Firefox 151 selects that LAN address: the Viewer resumes decoded pictures through the existing Browser fallback after @@ -704,11 +705,9 @@ connection; the Chromium Host control continues. The Host source is generated VP8 delivered through the real capture-process interface, not physical capture. Acceptance checks fresh rendered frames at the end, not accumulated frames from before the interruption. All controls retire their App, Browsers and interfaces. -These are ordinary-transport controls of the current App, not integrated -Magicsock or physical roaming evidence. A faster remote tunnel alone cannot -establish complete App handoff; local playback composition remains an adoption -boundary. Retain the existing failure owners and address policy until a bounded -alternative passes both Browser and Host/Viewer controls. +These are ordinary-transport controls, not integrated Magicsock or physical +roaming evidence. A faster remote tunnel alone cannot establish complete App +handoff; both the network and Browser playback legs must recover. The local restart control exposed a separate address-cache defect: Pion ICE v4.4.0's [wildcard UDP mux](https://github.com/pion/ice/blob/v4.4.0/udp_mux.go) @@ -721,10 +720,26 @@ The isolated regression covers restart and a new connection; restoring the former constructor fails on the stale candidate. Firefox also resumes fresh rendered frames on the same track after a standard ICE restart, including from the failed state, while a separately bound loopback sibling keeps receiving. -That browser control uses complete SDP through a lab-only restart entry point. -It establishes the underlying repair, not product local-bridge recovery: the -App's current local bridge still negotiates only once, and its existing failure -owners remain unchanged. +Replacing only the Native PeerConnection while retaining the Browser connection +instead reported connected without fresh decoded pictures in the same fixture; +it is not the accepted recovery mechanism. + +The 2.0 candidate now uses the existing bridge owner for one standard ICE restart +after a three-second disconnect grace, or immediately after failure. Initial and +restart negotiations use complete local SDP with the request ID fencing the answer; +there is no separate local trickle queue. The negotiation has an eight-second +deadline, including a five-second Native gathering bound. Retirement cancels it; +failure retains the existing Host/Viewer terminal owners. Paused sources need not +produce new frames to prove a transport connection. + +Matched complete App controls retain the same Firefox Host and Viewer media track +through the namespace address change. The Host keeps sharing and decoding, its +downstream Viewer keeps rendering, and the Native Viewer resumes fresh rendered +frames without replacing its route or falling back to Browser reception. +Chromium Host/Viewer controls retain their working loopback leg without a restart. +These controls use ordinary Pion and generated media. They close the reproduced local +bridge gap, not the remaining Magicsock composition, physical roaming or platform +acceptance boundaries. Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof diff --git a/docs/standards/presentation-lifecycle.md b/docs/standards/presentation-lifecycle.md index dc55a650c..c511e28ab 100644 --- a/docs/standards/presentation-lifecycle.md +++ b/docs/standards/presentation-lifecycle.md @@ -235,7 +235,13 @@ recovery state still requires a fresh current frame to clear. SFU media is retained independently of transient room-signaling loss, but Browser or OS suspension and page reclamation remain outside Web guarantees. -Failure of the local Native media bridge disables Native reception for the +The local Native media bridge owns bounded ICE recovery on its existing +connection and stream. Its offer identity fences each answer; retirement cancels +pending negotiation and cannot recreate a source or receiver. Connection recovery +does not assert fresh media, and an intentionally paused source is not a failure. +First-frame and playback owners retain their existing evidence requirements. + +Terminal failure of the local Native media bridge disables Native reception for the current Viewer session before route recovery runs. Future peers in that session use Browser reception; existing healthy peers are not torn down by this choice. Parent-network failure alone is not evidence that the local Native capability diff --git a/docs/todo.md b/docs/todo.md index 3df84315e..595f08993 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -76,13 +76,11 @@ Implementation needs a concrete useful result within the product model. Live-encoder/decoded-picture controls now separate shared forwarding queue delay from the local playback leg losing its selected LAN address. The shared send-queue repair passes constrained-link recovery. The complete App controls - confirm that its local Browser bridge is a separate handoff limit, especially - for Host preview. Resolve that composition within the existing owners before - claiming complete App recovery. A loopback-only policy fails the Firefox - control; retain the current address policy. Fresh interface gathering fixes - stale host candidates, and a lab-only standard ICE restart preserves the local - track. Product bridge renegotiation, its generation fence and bounded recovery - are still unimplemented; do not count that experiment as complete App recovery. + exposed a separate local Browser bridge limit. Fresh interface gathering and + bounded bridge renegotiation now preserve the local track in complete ordinary + Pion App controls, including Firefox Host and Viewer. Retain the existing + address policy: forcing loopback fails Firefox. Verify the combined Magicsock + and Browser pipeline before claiming complete App recovery through that transport. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) and RTCP/MTU boundaries, retained-flow recovery beyond the passing serial diff --git a/internal/app/mediaedge/edge.go b/internal/app/mediaedge/edge.go index ea79d8923..9caf9617d 100644 --- a/internal/app/mediaedge/edge.go +++ b/internal/app/mediaedge/edge.go @@ -1,12 +1,14 @@ package mediaedge import ( + "context" "errors" "io" "log/slog" "strconv" "strings" "sync" + "time" "github.com/TNTcraftHIM/Piik/internal/app/nativeaudio" "github.com/TNTcraftHIM/Piik/internal/diagnostics" @@ -51,6 +53,7 @@ type Edge struct { qualityMu sync.Mutex qualityBaseline qualityBaseline answerMu sync.Mutex + localOfferID string localCandidates *localCandidateGathering } @@ -143,15 +146,14 @@ func (engine *Engine) NewEdge(source *Source, options EdgeOptions) (_ *Edge, err return nil, err } } - edge.localCandidates = newLocalCandidateGathering( - socket, - options.ICEServers, - prepareMapping, - options.Events.LocalCandidate, - ) - connection.OnICECandidate(func(candidate *webrtc.ICECandidate) { - edge.localCandidates.addPion(candidate) - }) + if !options.Local { + edge.localCandidates = newLocalCandidateGathering( + socket, options.ICEServers, prepareMapping, options.Events.LocalCandidate, + ) + connection.OnICECandidate(func(candidate *webrtc.ICECandidate) { + edge.localCandidates.addPion(candidate) + }) + } connection.OnICEConnectionStateChange(func(state webrtc.ICEConnectionState) { log.Debug("piik-client", "event", "media-ice-state", "direction", "outbound", "local", options.Local, "state", state.String()) }) @@ -226,6 +228,9 @@ func (edge *Edge) SetTargetLayer(layer int) error { } func (edge *Edge) CreateOffer() (webrtc.SessionDescription, error) { + if edge.local { + return webrtc.SessionDescription{}, errors.New("local media requires an identified offer") + } offer, err := edge.connection.CreateOffer(nil) if err != nil { return webrtc.SessionDescription{}, err @@ -237,9 +242,54 @@ func (edge *Edge) CreateOffer() (webrtc.SessionDescription, error) { return offer, nil } -func (edge *Edge) SetAnswer(answer webrtc.SessionDescription) error { +// Local bridges have no STUN or gateway discovery. Complete SDP keeps their +// candidates and answer in one bounded negotiation, including ICE restarts. +func (edge *Edge) CreateLocalOffer(ctx context.Context, offerID string, restart bool) (webrtc.SessionDescription, error) { edge.answerMu.Lock() defer edge.answerMu.Unlock() + edge.mu.Lock() + available := edge.local && !edge.closed && offerID != "" && offerID != edge.localOfferID && + ((!restart && edge.localOfferID == "") || (restart && edge.remoteDescriptionSet)) + edge.mu.Unlock() + if !available { + return webrtc.SessionDescription{}, errors.New("local media negotiation is unavailable") + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := ctx.Err(); err != nil { + return webrtc.SessionDescription{}, err + } + offer, err := edge.connection.CreateOffer(&webrtc.OfferOptions{ICERestart: restart}) + if err != nil { + return webrtc.SessionDescription{}, err + } + gathered := webrtc.GatheringCompletePromise(edge.connection) + if err = edge.connection.SetLocalDescription(offer); err != nil { + return webrtc.SessionDescription{}, err + } + edge.localOfferID = offerID + edge.mu.Lock() + edge.remoteDescriptionSet = false + edge.mu.Unlock() + select { + case <-gathered: + if description := edge.connection.LocalDescription(); description != nil { + return *description, nil + } + return webrtc.SessionDescription{}, io.ErrClosedPipe + case <-ctx.Done(): + return webrtc.SessionDescription{}, ctx.Err() + case <-edge.socket.ctx.Done(): + return webrtc.SessionDescription{}, io.ErrClosedPipe + } +} + +func (edge *Edge) SetAnswer(answer webrtc.SessionDescription, offerID string) error { + edge.answerMu.Lock() + defer edge.answerMu.Unlock() + if (edge.local && (offerID == "" || offerID != edge.localOfferID)) || (!edge.local && offerID != "") { + return errors.New("native media answer does not match its offer") + } if answer.Type != webrtc.SDPTypeAnswer { return errors.New("native media edge requires an SDP answer") } @@ -249,9 +299,8 @@ func (edge *Edge) SetAnswer(answer webrtc.SessionDescription) error { return errors.New("native media edge is closed") } if edge.remoteDescriptionSet { - // The connection identity fences the answer to one edge. A repeated - // answer is a retransmission; the first applied description remains - // authoritative. + // A repeated answer within the same connection/offer is a + // retransmission; its first applied description remains authoritative. edge.mu.Unlock() return nil } @@ -280,6 +329,9 @@ func (edge *Edge) SetAnswer(answer webrtc.SessionDescription) error { } func (edge *Edge) AddRemoteCandidate(candidate *webrtc.ICECandidateInit) error { + if edge.local { + return errors.New("local media candidates belong in the SDP answer") + } if malformedRemoteCandidate(candidate) { slog.Debug("media-candidate-rejected", "connectionId", diagnostics.ID(edge.connectionID), "stage", "malformed") return nil diff --git a/internal/app/mediaedge/edge_test.go b/internal/app/mediaedge/edge_test.go index 95a4269b9..a4e78c2a4 100644 --- a/internal/app/mediaedge/edge_test.go +++ b/internal/app/mediaedge/edge_test.go @@ -467,7 +467,7 @@ func TestRepeatedAnswerIsIdempotent(t *testing.T) { if answer == nil { t.Fatal("receiver did not retain its answer") } - if err = edge.SetAnswer(*answer); err != nil { + if err = edge.SetAnswer(*answer, ""); err != nil { t.Fatalf("repeated answer was not idempotent: %v", err) } } @@ -741,7 +741,7 @@ func TestAudioSourceDeliversOpusOnTheVideoPeerConnection(t *testing.T) { t.Fatal(err) } answer.SDP = strings.ReplaceAll(answer.SDP, "maxaveragebitrate=510000", "maxaveragebitrate=192000") - if err = limited.SetAnswer(answer); err == nil { + if err = limited.SetAnswer(answer, ""); err == nil { t.Fatal("shared audio accepted an insufficient receiver limit") } if limited.connection.RemoteDescription() != nil { @@ -822,7 +822,7 @@ func connectEdgeToReceiver( t.Fatal(err) } waitSignal(t, gatherAnswer, "answer ICE gathering") - if err = edge.SetAnswer(*receiver.LocalDescription()); err != nil { + if err = edge.SetAnswer(*receiver.LocalDescription(), ""); err != nil { t.Fatal(err) } deadline := time.Now().Add(5 * time.Second) diff --git a/internal/app/mediaedge/local_edge_test.go b/internal/app/mediaedge/local_edge_test.go new file mode 100644 index 000000000..9d3584656 --- /dev/null +++ b/internal/app/mediaedge/local_edge_test.go @@ -0,0 +1,94 @@ +package mediaedge + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/pion/webrtc/v4" +) + +func TestLocalRestartRetainsMediaAndRejectsStaleAnswers(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) + check(err) + t.Cleanup(func() { _ = engine.Close() }) + source, err := engine.NewSource("vp8", 1, 1, nil) + check(err) + t.Cleanup(func() { _ = source.Close() }) + edge, err := engine.NewEdge(source, EdgeOptions{ConnectionID: "local-preview", Local: true}) + check(err) + t.Cleanup(func() { _ = edge.Close() }) + receiver := newReceiver(t) + t.Cleanup(func() { _ = receiver.Close() }) + connection, transport, sender := edge.connection, edge.transport, edge.sender + answer := func(offer webrtc.SessionDescription) webrtc.SessionDescription { + t.Helper() + check(receiver.SetRemoteDescription(offer)) + description, err := receiver.CreateAnswer(nil) + check(err) + gathered := webrtc.GatheringCompletePromise(receiver) + check(receiver.SetLocalDescription(description)) + select { + case <-gathered: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + return *receiver.LocalDescription() + } + ufrag := func(sdp string) string { + for _, line := range strings.Split(sdp, "\r\n") { + if strings.HasPrefix(line, "a=ice-ufrag:") { + return line + } + } + t.Fatal("missing ICE credentials") + return "" + } + initial, err := edge.CreateLocalOffer(ctx, "local_offer_1", false) + check(err) + if !strings.Contains(initial.SDP, "a=candidate:") || edge.localCandidates != nil { + t.Fatal("local bridge did not use complete SDP") + } + oldAnswer := answer(initial) + check(edge.SetAnswer(oldAnswer, "local_offer_1")) + check(edge.SetAnswer(oldAnswer, "local_offer_1")) // Same transaction is idempotent. + for _, offerID := range []string{"local_offer_2", "local_offer_3"} { + offer, err := edge.CreateLocalOffer(ctx, offerID, true) + check(err) + if ufrag(initial.SDP) == ufrag(offer.SDP) { + t.Fatal("restart reused ICE credentials") + } + if err = edge.SetAnswer(oldAnswer, "local_offer_1"); err == nil { + t.Fatal("stale answer was accepted into a newer negotiation") + } + if _, err = edge.CreateLocalOffer(ctx, "overlapping_offer", true); err == nil { + t.Fatal("an unanswered negotiation was replaced") + } + check(edge.SetAnswer(answer(offer), offerID)) + if edge.connection != connection || edge.transport != transport || edge.sender != sender || edge.source != source { + t.Fatal("ICE recovery replaced a media owner") + } + initial = offer + } + canceled, stop := context.WithCancel(ctx) + stop() + if _, err = edge.CreateLocalOffer(canceled, "canceled_offer", true); err == nil { + t.Fatal("canceled operation changed negotiation") + } + if edge.localOfferID != "local_offer_3" { + t.Fatal("canceled operation replaced offer identity") + } + check(edge.Close()) + if _, err = edge.CreateLocalOffer(ctx, "retired_offer", true); err == nil { + t.Fatal("restart reopened a retired edge") + } +} diff --git a/internal/app/mediaedge/publication.go b/internal/app/mediaedge/publication.go index 11ad63ba3..39613630e 100644 --- a/internal/app/mediaedge/publication.go +++ b/internal/app/mediaedge/publication.go @@ -117,7 +117,7 @@ func (publication *Publication) CreateOffer() (webrtc.SessionDescription, error) return publication.signaling.CreateOffer() } func (publication *Publication) SetAnswer(answer webrtc.SessionDescription) error { - return publication.signaling.SetAnswer(answer) + return publication.signaling.SetAnswer(answer, "") } func (publication *Publication) AddRemoteCandidate(candidate *webrtc.ICECandidateInit) error { return publication.signaling.AddRemoteCandidate(candidate) diff --git a/internal/app/mediaedge/receiver_overlap_test.go b/internal/app/mediaedge/receiver_overlap_test.go index 7b6b1fef2..082eb7d19 100644 --- a/internal/app/mediaedge/receiver_overlap_test.go +++ b/internal/app/mediaedge/receiver_overlap_test.go @@ -51,7 +51,7 @@ func TestNativeReceiverOverlapKeepsCurrentMedia(t *testing.T) { } t.Cleanup(func() { _ = receiver.Close() }) waitSignal(t, webrtc.GatheringCompletePromise(receiver.connection), "Native answer") - if err = edge.SetAnswer(*receiver.connection.LocalDescription()); err != nil { + if err = edge.SetAnswer(*receiver.connection.LocalDescription(), ""); err != nil { t.Fatal(err) } waitConnected(t, edge.connection, id) diff --git a/internal/app/mediaedge/relay_chain_fixture_test.go b/internal/app/mediaedge/relay_chain_fixture_test.go index 5d1170a8e..b15913eca 100644 --- a/internal/app/mediaedge/relay_chain_fixture_test.go +++ b/internal/app/mediaedge/relay_chain_fixture_test.go @@ -138,7 +138,7 @@ func TestRelayChainEncodedFixture(t *testing.T) { check(createErr) relays = append(relays, receiver) waitSignal(t, webrtc.GatheringCompletePromise(receiver.connection), "chain answer gathering") - check(edge.SetAnswer(*receiver.connection.LocalDescription())) + check(edge.SetAnswer(*receiver.connection.LocalDescription(), "")) waitConnected(t, edge.connection, "chain upstream") waitConnected(t, receiver.connection, "chain native receiver") check(receiver.Source().SetRelayProfile(nativecapture.VideoProfile{ diff --git a/internal/app/nativecontrol/session.go b/internal/app/nativecontrol/session.go index 74c488593..ca1af8cf7 100644 --- a/internal/app/nativecontrol/session.go +++ b/internal/app/nativecontrol/session.go @@ -57,7 +57,7 @@ type Session struct { } type outboundMediaSession interface { - SetAnswer(string, webrtc.SessionDescription) error + SetAnswer(string, webrtc.SessionDescription, string) error AddCandidate(string, *webrtc.ICECandidateInit) error CloseEdge(string) } @@ -355,7 +355,8 @@ func (session *Session) Handle(ctx context.Context, payload []byte) (result any, var request edgeAnswerRequest if err := decodeStrict(payload, &request); err != nil || request.Type != envelope.Type || - !validIdentities(request.ShareID, request.ConnectionID) { + !validIdentities(request.ShareID, request.ConnectionID) || + (request.OfferID != "" && !identityPattern.MatchString(request.OfferID)) { return nil, protocolViolation("native edge-answer request is invalid") } if !validSDP(request.SDP) { @@ -368,6 +369,7 @@ func (session *Session) Handle(ctx context.Context, payload []byte) (result any, if err := media.SetAnswer( request.ConnectionID, webrtc.SessionDescription{Type: webrtc.SDPTypeAnswer, SDP: request.SDP}, + request.OfferID, ); err != nil { return nil, err } @@ -696,12 +698,13 @@ func (session *Session) prepareLocalEdge( var offer webrtc.SessionDescription var err error if host := session.current(request.ShareID); host != nil && request.SourceConnectionID == "" { - offer, err = host.PrepareLocalEdge(request.ConnectionID) + offer, err = host.PrepareLocalEdge(request.ConnectionID, envelope.ID, request.Restart) } else if viewer := session.currentViewer(request.ShareID); viewer != nil && identityPattern.MatchString(request.SourceConnectionID) { offer, err = viewer.PrepareLocalEdge( request.SourceConnectionID, request.ConnectionID, + envelope.ID, request.Restart, ) } else { return nil, errors.New("native media source does not exist") diff --git a/internal/app/nativecontrol/wire.go b/internal/app/nativecontrol/wire.go index eddcda346..96da3830f 100644 --- a/internal/app/nativecontrol/wire.go +++ b/internal/app/nativecontrol/wire.go @@ -125,6 +125,7 @@ type prepareLocalEdgeRequest struct { ShareID string `json:"shareId"` ConnectionID string `json:"connectionId"` SourceConnectionID string `json:"sourceConnectionId,omitempty"` + Restart bool `json:"restart"` } type receiveOfferRequest struct { @@ -170,6 +171,7 @@ type edgeAnswerRequest struct { ShareID string `json:"shareId"` ConnectionID string `json:"connectionId"` SDP string `json:"sdp"` + OfferID string `json:"offerId,omitempty"` } type edgeCandidateRequest struct { diff --git a/internal/app/nativehost/audio_mix_test.go b/internal/app/nativehost/audio_mix_test.go index 1e72e84c1..318963bdc 100644 --- a/internal/app/nativehost/audio_mix_test.go +++ b/internal/app/nativehost/audio_mix_test.go @@ -171,7 +171,7 @@ func TestAudioExclusionFailureMutesOnlySourceAndNeverFallsBack(t *testing.T) { func TestSameVideoSourceAudioChangesPreserveCaptureAndConnections(t *testing.T) { session, options, _ := startMixedFixture(t) - if _, err := session.PrepareLocalEdge("audio_change"); err != nil { + if _, err := session.PrepareLocalEdge("audio_change", "local_offer_1", false); err != nil { t.Fatal(err) } video, source, output, edge := session.stream, session.source, session.audioSource, session.edge("audio_change") @@ -210,7 +210,7 @@ func TestNativeMicrophonePreservesMediaOwnersAcrossInputChanges(t *testing.T) { if !session.HasAudio() || session.SourceAudio() == nil || *session.SourceAudio() { t.Fatal("silent mixed output was not reserved independently of source") } - _, err := session.PrepareLocalEdge("mixed_edge") + _, err := session.PrepareLocalEdge("mixed_edge", "local_offer_1", false) check(err) edge, source := session.edge("mixed_edge"), session.audioSource enabled, gain := true, 1.0 diff --git a/internal/app/nativehost/audio_test.go b/internal/app/nativehost/audio_test.go index 51bd8725a..a51ed0966 100644 --- a/internal/app/nativehost/audio_test.go +++ b/internal/app/nativehost/audio_test.go @@ -147,7 +147,7 @@ func TestAudioEOFWaitsForExplicitSourceReplacement(t *testing.T) { } } }) - offer, err := session.PrepareLocalEdge("audio-edge") + offer, err := session.PrepareLocalEdge("audio-edge", "local_offer_1", false) check(err) check(receiver.SetRemoteDescription(offer)) // A receive preference belongs to this answer, not the upstream offer. @@ -184,7 +184,7 @@ func TestAudioEOFWaitsForExplicitSourceReplacement(t *testing.T) { case <-ctx.Done(): t.Fatal("receiver gathering timed out") } - check(session.SetAnswer("audio-edge", *receiver.LocalDescription())) + check(session.SetAnswer("audio-edge", *receiver.LocalDescription(), "local_offer_1")) waitPackets := func(counter *atomic.Uint64, minimum uint64, label string) { t.Helper() deadline := time.Now().Add(3 * time.Second) diff --git a/internal/app/nativehost/session.go b/internal/app/nativehost/session.go index 2fa37f301..4868a3b2c 100644 --- a/internal/app/nativehost/session.go +++ b/internal/app/nativehost/session.go @@ -235,19 +235,27 @@ func (session *Session) PrepareEdge( connectionID string, iceServers []webrtc.ICEServer, ) (webrtc.SessionDescription, error) { - return session.prepareEdge(connectionID, iceServers, false) + return session.prepareEdge(connectionID, iceServers, "") } func (session *Session) PrepareLocalEdge( - connectionID string, + connectionID, offerID string, + restart bool, ) (webrtc.SessionDescription, error) { - return session.prepareEdge(connectionID, nil, true) + if restart { + edge := session.edge(connectionID) + if edge == nil { + return webrtc.SessionDescription{}, errors.New("native local media edge does not exist") + } + return edge.CreateLocalOffer(session.ctx, offerID, true) + } + return session.prepareEdge(connectionID, nil, offerID) } func (session *Session) prepareEdge( connectionID string, iceServers []webrtc.ICEServer, - local bool, + localOfferID string, ) (webrtc.SessionDescription, error) { session.mu.Lock() if session.closed || session.edges[connectionID] != nil { @@ -259,7 +267,7 @@ func (session *Session) prepareEdge( ConnectionID: connectionID, ICEServers: iceServers, Audio: session.audioSource, - Local: local, + Local: localOfferID != "", Events: mediaedge.EdgeEvents{ LocalCandidate: func(candidate *webrtc.ICECandidateInit) { session.emit(Event{ @@ -296,7 +304,12 @@ func (session *Session) prepareEdge( } session.edges[connectionID] = edge session.mu.Unlock() - offer, err := edge.CreateOffer() + var offer webrtc.SessionDescription + if localOfferID != "" { + offer, err = edge.CreateLocalOffer(session.ctx, localOfferID, false) + } else { + offer, err = edge.CreateOffer() + } if err != nil { session.CloseEdge(connectionID) return webrtc.SessionDescription{}, err @@ -304,12 +317,12 @@ func (session *Session) prepareEdge( return offer, nil } -func (session *Session) SetAnswer(connectionID string, answer webrtc.SessionDescription) error { +func (session *Session) SetAnswer(connectionID string, answer webrtc.SessionDescription, offerID string) error { edge := session.edge(connectionID) if edge == nil { return errors.New("native media edge does not exist") } - return edge.SetAnswer(answer) + return edge.SetAnswer(answer, offerID) } func (session *Session) AddCandidate( @@ -709,7 +722,8 @@ func (session *Session) run() { }() if session.mixer != nil { go func() { session.mixer.run(); close(audioDone) }() - } else if session.audioStream == nil { + } else if session.audioSource == nil { + // The session's output shape is fixed; source inputs can change concurrently. close(audioDone) } else { go func() { diff --git a/internal/app/nativeviewer/session.go b/internal/app/nativeviewer/session.go index c48ad5d6c..a2aa88cb7 100644 --- a/internal/app/nativeviewer/session.go +++ b/internal/app/nativeviewer/session.go @@ -203,9 +203,20 @@ func (session *Session) AddReceiverCandidate( func (session *Session) PrepareLocalEdge( sourceConnectionID string, - connectionID string, + connectionID, offerID string, + restart bool, ) (webrtc.SessionDescription, error) { - return session.prepareEdge(sourceConnectionID, connectionID, nil, true) + if restart { + session.mu.Lock() + current := session.edges[connectionID] + available := !session.closed && current.edge != nil && current.sourceConnectionID == sourceConnectionID + session.mu.Unlock() + if !available { + return webrtc.SessionDescription{}, errors.New("native local Viewer edge does not exist") + } + return current.edge.CreateLocalOffer(session.ctx, offerID, true) + } + return session.prepareEdge(sourceConnectionID, connectionID, nil, offerID) } func (session *Session) PrepareEdge( @@ -213,15 +224,16 @@ func (session *Session) PrepareEdge( connectionID string, iceServers []webrtc.ICEServer, ) (webrtc.SessionDescription, error) { - return session.prepareEdge(sourceConnectionID, connectionID, iceServers, false) + return session.prepareEdge(sourceConnectionID, connectionID, iceServers, "") } func (session *Session) prepareEdge( sourceConnectionID string, connectionID string, iceServers []webrtc.ICEServer, - local bool, + localOfferID string, ) (webrtc.SessionDescription, error) { + local := localOfferID != "" receiver := session.receiver(sourceConnectionID) if receiver == nil { return webrtc.SessionDescription{}, errors.New("native Viewer source does not exist") @@ -279,7 +291,12 @@ func (session *Session) prepareEdge( edge: edge, sourceConnectionID: sourceConnectionID, } session.mu.Unlock() - offer, err := edge.CreateOffer() + var offer webrtc.SessionDescription + if local { + offer, err = edge.CreateLocalOffer(session.ctx, localOfferID, false) + } else { + offer, err = edge.CreateOffer() + } if err != nil { session.CloseEdge(connectionID) return webrtc.SessionDescription{}, err @@ -287,12 +304,12 @@ func (session *Session) prepareEdge( return offer, nil } -func (session *Session) SetAnswer(connectionID string, answer webrtc.SessionDescription) error { +func (session *Session) SetAnswer(connectionID string, answer webrtc.SessionDescription, offerID string) error { edge := session.edge(connectionID) if edge == nil { return errors.New("native Viewer edge does not exist") } - return edge.SetAnswer(answer) + return edge.SetAnswer(answer, offerID) } func (session *Session) AddCandidate(connectionID string, candidate *webrtc.ICECandidateInit) error { diff --git a/internal/app/nativeviewer/session_test.go b/internal/app/nativeviewer/session_test.go index b5aafbe5b..5123c12b5 100644 --- a/internal/app/nativeviewer/session_test.go +++ b/internal/app/nativeviewer/session_test.go @@ -47,7 +47,7 @@ func TestOfferReuseKeepsSourceAndChildOwnership(t *testing.T) { t.Fatal("incompatible source retained its old edge") } receiver := session.receiver("upstream") - if _, err = session.PrepareLocalEdge("upstream", "local-playback"); err != nil { + if _, err = session.PrepareLocalEdge("upstream", "local-playback", "local_offer_1", false); err != nil { t.Fatal(err) } child := session.edge("local-playback") diff --git a/src/client/native/client.ts b/src/client/native/client.ts index 84ae94de3..81e04d8c5 100644 --- a/src/client/native/client.ts +++ b/src/client/native/client.ts @@ -501,12 +501,14 @@ export class NativeClient { shareId: string, connectionId: string, sourceConnectionId?: string, - ): Promise { + restart = false, + ): Promise { const response = await this.request( "prepare-local-edge", { shareId, connectionId, + restart, ...(sourceConnectionId ? { sourceConnectionId } : {}), }, edgeOfferResponseSchema, @@ -517,7 +519,7 @@ export class NativeClient { ) { throw new Error("Native local edge identity changed"); } - return { type: "offer", sdp: response.sdp }; + return { type: "offer", sdp: response.sdp, offerId: response.id }; } async receiveOffer( @@ -591,13 +593,14 @@ export class NativeClient { shareId: string, connectionId: string, payload: SignalPayload, + offerId?: string, ): Promise { if (payload.connectionId !== connectionId) return; if (payload.kind === "description") { if (payload.description.type !== "answer") return; await this.request( "edge-answer", - { shareId, connectionId, sdp: payload.description.sdp }, + { shareId, connectionId, sdp: payload.description.sdp, ...(offerId ? { offerId } : {}) }, nativeAckResponseSchema, ); return; diff --git a/src/client/native/media-bridge.ts b/src/client/native/media-bridge.ts index dcaf209f5..b04db9867 100644 --- a/src/client/native/media-bridge.ts +++ b/src/client/native/media-bridge.ts @@ -3,7 +3,6 @@ import { createOpaqueId } from "../lib/opaque-id"; import { browserDebugEnabled, debugError, debugEvent } from "../lib/debug"; import { observeDebugConnection } from "../lib/debug-webrtc"; import type { ConnectionMetrics } from "../types"; -import { addRemoteIceCandidate, type SignalCandidate } from "../webrtc/nat-prediction"; import { preferScreenAudioStereo } from "../webrtc/screen-audio-sdp"; import { collectConnectionMetrics, @@ -13,7 +12,7 @@ import { import type { NativeClientEvent } from "./wire"; const BRIDGE_TIMEOUT_MS = 8_000; -const MAX_PENDING_CANDIDATES = 64; +const DISCONNECTED_GRACE_MS = 3_000; export class NativeMediaBridgeError extends Error {} export class NativeMediaBridgeInitializationError extends NativeMediaBridgeError {} @@ -23,11 +22,13 @@ export interface NativeMediaBridgeControl { shareId: string, connectionId: string, sourceConnectionId?: string, - ): Promise; + restart?: boolean, + ): Promise; acceptSignal( shareId: string, connectionId: string, payload: SignalPayload, + offerId?: string, ): Promise; closeEdge(shareId: string, connectionId: string): Promise; onEvent(listener: (event: NativeClientEvent) => void): () => void; @@ -38,14 +39,16 @@ export class NativeMediaBridge { readonly connectionId = createOpaqueId(); private readonly peer: RTCPeerConnection; - private readonly pendingCandidates: SignalCandidate[] = []; + private readonly lifetime = new AbortController(); private unsubscribe: (() => void) | null = null; private edgeRequested = false; - private remoteDescriptionSet = false; private ready = false; private disposed = false; - private startTimer: number | null = null; - private rejectStart: ((error: Error) => void) | null = null; + private expectedAudio = false; + private deadline: number | null = null; + private disconnectTimer: number | null = null; + private completeNegotiation: (() => void) | null = null; + private rejectNegotiation: ((error: Error) => void) | null = null; constructor( private readonly shareId: string, @@ -81,39 +84,16 @@ export class NativeMediaBridge { if (this.disposed || this.unsubscribe) { throw new NativeMediaBridgeError("Native media bridge is unavailable"); } - const started = new Promise((resolve, reject) => { - this.rejectStart = reject; - this.startTimer = window.setTimeout(() => this.fail("timeout"), BRIDGE_TIMEOUT_MS); - const complete = () => { - if ( - !this.ready && - this.peer.connectionState === "connected" && - this.stream.getVideoTracks().length > 0 && - (!expectedAudio || this.stream.getAudioTracks().length > 0) - ) { - this.ready = true; - this.rejectStart = null; - this.clearStartTimer(); - resolve(this.stream); - } - }; - this.peer.addEventListener("track", (event) => { - if (!this.stream.getTracks().includes(event.track)) { - this.stream.addTrack(event.track); - } - complete(); - }); - this.peer.addEventListener("connectionstatechange", () => { - debugEvent("native-bridge", "connection-state", { state: this.peer.connectionState }); - if ( - this.peer.connectionState === "failed" || - this.peer.connectionState === "closed" - ) { - this.fail("connection-state"); - return; - } - complete(); - }); + this.expectedAudio = expectedAudio; + this.peer.addEventListener("track", (event) => { + if (this.disposed) return; + if (!this.stream.getTracks().includes(event.track)) this.stream.addTrack(event.track); + this.completeNegotiation?.(); + }); + this.peer.addEventListener("connectionstatechange", () => { + debugEvent("native-bridge", "connection-state", { state: this.peer.connectionState }); + this.connectionStateChanged(this.peer.connectionState); + this.completeNegotiation?.(); }); this.peer.addEventListener("iceconnectionstatechange", () => { debugEvent("native-bridge", "ice-state", { state: this.peer.iceConnectionState }); @@ -121,25 +101,6 @@ export class NativeMediaBridge { this.peer.addEventListener("icegatheringstatechange", () => { debugEvent("native-bridge", "ice-gathering-state", { state: this.peer.iceGatheringState }); }); - this.peer.addEventListener("icecandidate", (event) => { - if (this.disposed) return; - void this.control.acceptSignal( - this.shareId, - this.connectionId, - { - kind: "candidate", - connectionId: this.connectionId, - candidate: event.candidate - ? { - candidate: event.candidate.candidate, - sdpMid: event.candidate.sdpMid, - sdpMLineIndex: event.candidate.sdpMLineIndex, - usernameFragment: event.candidate.usernameFragment, - } - : null, - }, - ).catch((error) => this.fail("candidate-signal", error)); - }); this.unsubscribe = this.control.onEvent((event) => { if (event.shareId !== this.shareId) return; if (event.type === "share-ended") { @@ -152,77 +113,22 @@ export class NativeMediaBridge { ) { return; } - if (event.type === "edge-candidate") { - if (!this.remoteDescriptionSet) { - if (this.pendingCandidates.length >= MAX_PENDING_CANDIDATES) { - this.fail("candidate-overflow"); - } else { - this.pendingCandidates.push(event.candidate); - } - return; - } - void addRemoteIceCandidate(this.peer, event.candidate) - .catch((error) => this.fail("candidate-apply", error)); - } else if ( - event.type === "edge-state" && - (event.state === "failed" || event.state === "closed") - ) { - this.fail("native-state"); - } + if (event.type === "edge-state") this.connectionStateChanged(event.state); }); - - const negotiate = async () => { - this.edgeRequested = true; - const offer = await this.control.prepareLocalEdge( - this.shareId, - this.connectionId, - this.sourceConnectionId, - ); - if (this.disposed) return await started; - await this.peer.setRemoteDescription(offer); - this.remoteDescriptionSet = true; - for (const candidate of this.pendingCandidates.splice(0)) { - await addRemoteIceCandidate(this.peer, candidate); - } - const answer = preferScreenAudioStereo(await this.peer.createAnswer()); - if (!answer.sdp) { - throw new Error("Native media bridge produced no SDP answer"); - } - await this.peer.setLocalDescription(answer); - if (this.disposed) return await started; - await this.control.acceptSignal( - this.shareId, - this.connectionId, - { - kind: "description", - connectionId: this.connectionId, - description: { - type: "answer", - sdp: answer.sdp, - }, - }, - ); - return await started; - }; - try { - return await Promise.race([negotiate(), started]); - } catch (error) { - this.fail("negotiation", error); - throw error instanceof NativeMediaBridgeError - ? error - : new NativeMediaBridgeError("Native media bridge failed", { cause: error }); - } + return this.negotiate(false); } dispose(): void { if (this.disposed) return; this.disposed = true; - this.rejectStart?.(new NativeMediaBridgeError("Native media bridge failed")); - this.rejectStart = null; + this.lifetime.abort(); + this.rejectNegotiation?.(new NativeMediaBridgeError("Native media bridge failed")); + this.rejectNegotiation = null; + this.completeNegotiation = null; this.unsubscribe?.(); this.unsubscribe = null; - this.clearStartTimer(); - this.pendingCandidates.length = 0; + this.clearDeadline(); + this.clearDisconnectTimer(); this.peer.close(); this.stream.getTracks().forEach((track) => track.stop()); if (this.edgeRequested) { @@ -232,12 +138,102 @@ export class NativeMediaBridge { } } + private connectionStateChanged(state: RTCPeerConnectionState): void { + if (this.disposed) return; + if (state === "closed" || (!this.ready && state === "failed")) { + this.fail("connection-state"); + return; + } + if (state === "connected") { + if (this.peer.connectionState === "connected") this.clearDisconnectTimer(); + return; + } + if (!this.ready || this.completeNegotiation) return; + const restart = () => { + this.clearDisconnectTimer(); + if (this.disposed || this.completeNegotiation) return; + debugEvent("native-bridge", "restart"); + void this.negotiate(true).catch(() => undefined); + }; + if (state === "failed") restart(); + else if (state === "disconnected" && this.disconnectTimer === null) { + this.disconnectTimer = window.setTimeout(restart, DISCONNECTED_GRACE_MS); + } + } + + private negotiate(restart: boolean): Promise { + return new Promise((resolve, reject) => { + let signaled = false; + this.rejectNegotiation = reject; + this.deadline = window.setTimeout(() => this.fail("timeout"), BRIDGE_TIMEOUT_MS); + this.completeNegotiation = () => { + if (!signaled || this.disposed || this.peer.connectionState !== "connected" || + this.stream.getVideoTracks().length === 0 || + (this.expectedAudio && this.stream.getAudioTracks().length === 0)) return; + // This proves the local transport. First-frame/playback owners still + // prove media progress; an intentionally paused source can remain quiet. + this.ready = true; + this.completeNegotiation = null; + this.rejectNegotiation = null; + this.clearDeadline(); + resolve(this.stream); + }; + void this.exchangeDescriptions(restart).then(() => { + signaled = true; + this.completeNegotiation?.(); + }).catch(error => this.fail("negotiation", error)); + }); + } + + private async exchangeDescriptions(restart: boolean): Promise { + this.edgeRequested = true; + const { offerId, ...offer } = await this.control.prepareLocalEdge( + this.shareId, this.connectionId, this.sourceConnectionId, restart, + ); + this.lifetime.signal.throwIfAborted(); + await this.peer.setRemoteDescription(offer); + this.lifetime.signal.throwIfAborted(); + const answer = preferScreenAudioStereo(await this.peer.createAnswer()); + this.lifetime.signal.throwIfAborted(); + await this.peer.setLocalDescription(answer); + await this.gatherCandidates(); + this.lifetime.signal.throwIfAborted(); + const sdp = this.peer.localDescription?.sdp; + if (!sdp) throw new Error("Native media bridge produced no SDP answer"); + await this.control.acceptSignal(this.shareId, this.connectionId, { + kind: "description", connectionId: this.connectionId, + description: { type: "answer", sdp }, + }, offerId); + } + + private gatherCandidates(): Promise { + const signal = this.lifetime.signal; + signal.throwIfAborted(); + if (this.peer.iceGatheringState === "complete") return Promise.resolve(); + return new Promise((resolve, reject) => { + const cleanup = () => { + this.peer.removeEventListener("icegatheringstatechange", complete); + signal.removeEventListener("abort", aborted); + }; + const complete = () => { + if (this.peer.iceGatheringState !== "complete") return; + cleanup(); + resolve(); + }; + const aborted = () => { cleanup(); reject(signal.reason); }; + this.peer.addEventListener("icegatheringstatechange", complete); + signal.addEventListener("abort", aborted, { once: true }); + }); + } + private fail(reason: string, error?: unknown): void { if (this.disposed) return; const details = { type: reason, state: this.ready ? "active" : "starting" }; if (error === undefined) debugEvent("native-bridge", "failed", details); else debugError("native-bridge", "failed", error, details); void this.reportFailureDiagnostics(); + this.rejectNegotiation?.(new NativeMediaBridgeError("Native media bridge failed", { cause: error })); + this.rejectNegotiation = null; if (this.ready) this.onFailed(); this.dispose(); } @@ -276,9 +272,15 @@ export class NativeMediaBridge { } } - private clearStartTimer(): void { - if (this.startTimer === null) return; - window.clearTimeout(this.startTimer); - this.startTimer = null; + private clearDeadline(): void { + if (this.deadline === null) return; + window.clearTimeout(this.deadline); + this.deadline = null; + } + + private clearDisconnectTimer(): void { + if (this.disconnectTimer === null) return; + window.clearTimeout(this.disconnectTimer); + this.disconnectTimer = null; } } diff --git a/tests/native-client.test.ts b/tests/native-client.test.ts index 1879b9674..180f776c5 100644 --- a/tests/native-client.test.ts +++ b/tests/native-client.test.ts @@ -39,6 +39,47 @@ describe("native App private wire", () => { expect(nativeHealthSchema.safeParse({ ...health, nativeMedia: { ...nativeMedia, maxVideoMacroblocksPerSecond: -1 } }).success).toBe(false); }); + it("binds a local answer to its prepare request across an ICE restart", async () => { + const requests: Record[] = []; + class Socket extends EventTarget { + static readonly OPEN = 1; + static readonly CLOSING = 2; + readyState = Socket.OPEN; + protocol = `piik-client-v10.${health.instanceToken}`; + constructor() { super(); queueMicrotask(() => this.dispatchEvent(new Event("open"))); } + close() { this.readyState = Socket.CLOSING; } + send(payload: string) { + const request = JSON.parse(payload) as Record; + requests.push(request); + const fields = request.type === "hello" ? { type: "ready" } + : request.type === "prepare-local-edge" ? { + type: "edge-offer", shareId: request.shareId, connectionId: request.connectionId, sdp: "v=0\r\n", + } : { type: "edge-answer-accepted" }; + queueMicrotask(() => this.dispatchEvent(new MessageEvent("message", { + data: JSON.stringify({ version: 10, id: request.id, ...fields }), + }))); + } + } + vi.stubGlobal("WebSocket", Socket); + vi.stubGlobal("window", { setTimeout, clearTimeout }); + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify(health)))); + const client = (await NativeClient.connect())!; + try { + let lastOffer = ""; + for (const restart of [false, true]) { + const offer = await client.prepareLocalEdge("share_123456", "local_123456", "source_123456", restart); + expect(offer.offerId).toBe(requests.at(-1)?.id); + expect(offer.offerId).not.toBe(lastOffer); + expect(requests.at(-1)).toMatchObject({ type: "prepare-local-edge", restart, sourceConnectionId: "source_123456" }); + await client.acceptSignal("share_123456", "local_123456", { + kind: "description", connectionId: "local_123456", description: { type: "answer", sdp: "v=0\r\n" }, + }, offer.offerId); + expect(requests.at(-1)).toMatchObject({ type: "edge-answer", offerId: offer.offerId }); + lastOffer = offer.offerId; + } + } finally { client.close(); } + }); + it.each(["prompt", "denied"] as const)("does not hold Viewer reception for %s permission and observes a later grant", async (state) => { vi.stubGlobal("window", { setTimeout, clearTimeout, location: { hostname: "piik.example" } }); const query = vi.fn().mockResolvedValue({ state }); diff --git a/tests/native-media-bridge.test.ts b/tests/native-media-bridge.test.ts index da1273ee7..52efbc2b8 100644 --- a/tests/native-media-bridge.test.ts +++ b/tests/native-media-bridge.test.ts @@ -38,7 +38,8 @@ class FakeMediaStream { class FakePeerConnection extends EventTarget { connectionState: RTCPeerConnectionState = "new"; iceConnectionState: RTCIceConnectionState = "new"; - iceGatheringState: RTCIceGatheringState = "new"; + iceGatheringState: RTCIceGatheringState = "complete"; + localDescription: RTCSessionDescriptionInit | null = null; readonly getReceivers = vi.fn<() => RTCRtpReceiver[]>(() => []); readonly getSenders = vi.fn<() => RTCRtpSender[]>(() => []); readonly getStats = vi.fn(async () => new Map()); @@ -48,7 +49,9 @@ class FakePeerConnection extends EventTarget { type: "answer" as const, sdp: "v=0\r\n", })); - readonly setLocalDescription = vi.fn(async (): Promise => undefined); + readonly setLocalDescription = vi.fn(async (description: RTCSessionDescriptionInit): Promise => { + this.localDescription = description; + }); readonly close = vi.fn(() => { this.connectionState = "closed"; }); @@ -78,20 +81,11 @@ function fixture(options: { }); const prepareLocalEdge = vi.fn< NativeMediaBridgeControl["prepareLocalEdge"] - >(async (_shareId, connectionId) => { - listener?.({ - version: 10, - type: "edge-candidate", - shareId: "share_123456", - connectionId, - candidate: { - candidate: "candidate:1 1 udp 1 127.0.0.1 9 typ host", - }, - }); + >(async () => { if (options.hangPreparation) { - return await new Promise(() => undefined); + return await new Promise(() => undefined); } - return { type: "offer", sdp: "v=0\r\n" }; + return { type: "offer", sdp: "v=0\r\n", offerId: "local_offer_1" }; }); const control: NativeMediaBridgeControl = { prepareLocalEdge, @@ -130,6 +124,82 @@ afterEach(() => { }); describe("native media bridge", () => { + it("recovers one retained transport after a disconnect grace, including a quiet source", async () => { + vi.useFakeTimers(); + const current = fixture(); + const starting = current.bridge.start(true); + const video = { kind: "video", stop: vi.fn() } as unknown as MediaStreamTrack; + const audio = { kind: "audio", stop: vi.fn() } as unknown as MediaStreamTrack; + current.peer().emitTrack(video); + current.peer().emitTrack(audio); + current.peer().setState("connected"); + await starting; + + current.peer().setState("disconnected"); + await vi.advanceTimersByTimeAsync(2_000); + current.peer().setState("connected"); + await vi.advanceTimersByTimeAsync(3_000); + expect(current.control.prepareLocalEdge).toHaveBeenCalledTimes(1); + + vi.mocked(current.control.prepareLocalEdge).mockResolvedValueOnce({ + type: "offer", sdp: "v=0\r\n", offerId: "local_offer_2", + }); + current.peer().setState("disconnected"); + await vi.advanceTimersByTimeAsync(3_000); + current.emit({ version: 10, type: "edge-state", shareId: "share_123456", + connectionId: current.bridge.connectionId, state: "failed" }); + expect(current.control.prepareLocalEdge).toHaveBeenLastCalledWith( + "share_123456", current.bridge.connectionId, undefined, true, + ); + expect(current.control.prepareLocalEdge).toHaveBeenCalledTimes(2); + expect(current.control.acceptSignal).toHaveBeenLastCalledWith( + "share_123456", current.bridge.connectionId, expect.objectContaining({ kind: "description" }), "local_offer_2", + ); + current.peer().setState("connected"); + await vi.advanceTimersByTimeAsync(8_000); + expect(current.bridge.stream.getTracks()).toEqual([video, audio]); + expect(video.stop).not.toHaveBeenCalled(); + expect(audio.stop).not.toHaveBeenCalled(); + expect(current.peer().close).not.toHaveBeenCalled(); + expect(current.onFailed).not.toHaveBeenCalled(); + current.bridge.dispose(); + }); + + it("bounds a failed restart and ignores its late offer after disposal", async () => { + vi.useFakeTimers(); + const current = fixture(); + const starting = current.bridge.start(); + current.peer().emitTrack({ kind: "video", stop: vi.fn() } as unknown as MediaStreamTrack); + current.peer().setState("connected"); + await starting; + let finish!: (offer: RTCSessionDescriptionInit & { offerId: string }) => void; + vi.mocked(current.control.prepareLocalEdge).mockImplementationOnce(() => new Promise(resolve => { finish = resolve; })); + current.peer().setState("failed"); + current.peer().setState("failed"); + expect(current.control.prepareLocalEdge).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(8_000); + finish({ type: "offer", sdp: "v=0\r\n", offerId: "late_offer" }); + await vi.advanceTimersByTimeAsync(1); + expect(current.onFailed).toHaveBeenCalledOnce(); + expect(current.peer().setRemoteDescription).toHaveBeenCalledOnce(); + expect(current.control.acceptSignal).toHaveBeenCalledOnce(); + expect(current.control.closeEdge).toHaveBeenCalledOnce(); + }); + + it("gathers the answer before signaling and cancels gathering on disposal", async () => { + const current = fixture(); + current.peer().iceGatheringState = "gathering"; + const starting = current.bridge.start(); + const rejected = expect(starting).rejects.toBeInstanceOf(NativeMediaBridgeError); + await vi.waitFor(() => expect(current.peer().setLocalDescription).toHaveBeenCalledOnce()); + expect(current.control.acceptSignal).not.toHaveBeenCalled(); + current.bridge.dispose(); + current.peer().iceGatheringState = "complete"; + current.peer().dispatchEvent(new Event("icegatheringstatechange")); + await rejected; + expect(current.control.acceptSignal).not.toHaveBeenCalled(); + }); + it("advertises the same Opus receive range on local preview as remote viewers", async () => { const current = fixture(); current.peer().createAnswer.mockResolvedValueOnce({ type: "answer", sdp: @@ -143,7 +213,7 @@ describe("native media bridge", () => { kind: "description", description: expect.objectContaining({ sdp: expect.stringContaining("stereo=1;maxaveragebitrate=510000"), }), - }), + }), "local_offer_1", )); expect(current.peer().setLocalDescription).toHaveBeenCalledWith(expect.objectContaining({ sdp: expect.stringContaining("stereo=1;maxaveragebitrate=510000"), @@ -197,107 +267,6 @@ describe("native media bridge", () => { expect(current.onFailed).not.toHaveBeenCalled(); }); - it.each(["candidate-signal", "candidate-apply"])("preserves the active %s failure and retires once", async (stage) => { - const current = fixture(); - const starting = current.bridge.start(); - await vi.waitFor(() => expect(current.peer().setLocalDescription).toHaveBeenCalledOnce()); - current.peer().emitTrack({ kind: "video", stop: vi.fn() } as unknown as MediaStreamTrack); - current.peer().setState("connected"); - await starting; - const cause = new DOMException("Connection is unavailable", "InvalidStateError"); - if (stage === "candidate-signal") { - vi.mocked(current.control.acceptSignal).mockRejectedValueOnce(cause); - current.peer().dispatchEvent(Object.assign(new Event("icecandidate"), { candidate: null })); - } else { - current.peer().addIceCandidate.mockRejectedValueOnce(cause); - current.emit({ version: 10, type: "edge-candidate", shareId: "share_123456", - connectionId: current.bridge.connectionId, candidate: null }); - } - await vi.waitFor(() => expect(current.onFailed).toHaveBeenCalledOnce()); - expect(debugError).toHaveBeenCalledWith("native-bridge", "failed", cause, { type: stage, state: "active" }); - expect(current.peer().close).toHaveBeenCalledOnce(); - expect(current.control.closeEdge).toHaveBeenCalledOnce(); - }); - - it.each(["queued", "active"])("keeps media alive after a %s candidate is rejected", async (phase) => { - const current = fixture(); - const peer = current.peer(); - if (phase === "queued") { - peer.addIceCandidate.mockRejectedValueOnce(new DOMException("Rejected candidate", "OperationError")); - } - const starting = current.bridge.start(); - // Register a rejection observer before allowing negotiation to run. - const result = starting.catch((error: unknown) => error); - if (phase === "queued") { - current.emit({ - version: 10, type: "edge-candidate", shareId: "share_123456", - connectionId: current.bridge.connectionId, - candidate: { candidate: "candidate:2 1 udp 1 127.0.0.1 10 typ host" }, - }); - } - await vi.waitFor(() => expect(peer.setLocalDescription).toHaveBeenCalledOnce()); - const track = { kind: "video", stop: vi.fn() } as unknown as MediaStreamTrack; - peer.emitTrack(track); - peer.setState("connected"); - await expect(result).resolves.toBe(current.bridge.stream); - if (phase === "active") { - peer.addIceCandidate.mockRejectedValueOnce(new DOMException("Rejected candidate", "OperationError")); - current.emit({ - version: 10, type: "edge-candidate", shareId: "share_123456", - connectionId: current.bridge.connectionId, - candidate: { candidate: "candidate:2 1 udp 1 127.0.0.1 10 typ host" }, - }); - } - current.emit({ - version: 10, type: "edge-candidate", shareId: "share_123456", - connectionId: current.bridge.connectionId, candidate: null, - }); - await vi.waitFor(() => expect(debugError).toHaveBeenCalledWith( - "webrtc", "remote-candidate-rejected", expect.any(DOMException), { origin: "ordinary" }, - )); - expect(peer.addIceCandidate).toHaveBeenCalledWith(null); - expect(peer.close).not.toHaveBeenCalled(); - expect(track.stop).not.toHaveBeenCalled(); - expect(current.onFailed).not.toHaveBeenCalled(); - current.bridge.dispose(); - }); - - it("still rejects invalid negotiation state when applying a candidate", async () => { - const current = fixture(); - current.peer().addIceCandidate.mockRejectedValueOnce(new DOMException("No description", "InvalidStateError")); - await expect(current.bridge.start()).rejects.toBeInstanceOf(NativeMediaBridgeError); - expect(current.peer().close).toHaveBeenCalledOnce(); - expect(current.control.closeEdge).toHaveBeenCalledOnce(); - }); - - it("orders an early native candidate after the remote offer", async () => { - const current = fixture(); - const starting = current.bridge.start(); - await vi.waitFor(() => { - expect(current.peer().setLocalDescription).toHaveBeenCalledOnce(); - }); - const track = { - kind: "video", - stop: vi.fn(), - } as unknown as MediaStreamTrack; - current.peer().emitTrack(track); - current.peer().setState("connected"); - - await expect(starting).resolves.toBe(current.bridge.stream); - expect(current.peer().addIceCandidate).toHaveBeenCalledWith( - expect.objectContaining({ candidate: expect.stringContaining("127.0.0.1") }), - ); - expect(current.control.acceptSignal).toHaveBeenCalledWith( - "share_123456", - current.bridge.connectionId, - expect.objectContaining({ kind: "description" }), - ); - current.bridge.dispose(); - expect(current.peer().close).toHaveBeenCalledOnce(); - expect(track.stop).toHaveBeenCalledOnce(); - expect(current.control.closeEdge).toHaveBeenCalledOnce(); - }); - it("releases a local edge when preparation exceeds the total deadline", async () => { vi.useFakeTimers(); const current = fixture({ hangPreparation: true }); @@ -389,7 +358,7 @@ describe("native media bridge", () => { ); }); - it("reports an active native failure once and retires its edge", async () => { + it("retires an actively closed native edge once", async () => { const current = fixture(); const starting = current.bridge.start(); await vi.waitFor(() => { @@ -407,14 +376,14 @@ describe("native media bridge", () => { type: "edge-state", shareId: "share_123456", connectionId: current.bridge.connectionId, - state: "failed", + state: "closed", }); current.emit({ version: 10, type: "edge-state", shareId: "share_123456", connectionId: current.bridge.connectionId, - state: "failed", + state: "closed", }); expect(current.onFailed).toHaveBeenCalledOnce(); @@ -445,7 +414,7 @@ describe("native media bridge", () => { if (stage === "prepare") { vi.mocked(current.control.prepareLocalEdge).mockImplementationOnce(async () => { await pending; - return { type: "offer", sdp: "v=0\r\n" }; + return { type: "offer", sdp: "v=0\r\n", offerId: "local_offer_1" }; }); } else { current.peer().setLocalDescription.mockImplementationOnce(() => pending); From 6436e4a53e7ecf06d818f58a44d8e7e6afcfc87b Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 04:30:55 +0800 Subject: [PATCH 092/262] docs(research): record combined browser transport evidence --- docs/research/nat-traversal.md | 17 +++++++++++++++++ docs/todo.md | 7 +++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index c3f578dca..fd7368fb0 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -741,6 +741,23 @@ These controls use ordinary Pion and generated media. They close the reproduced bridge gap, not the remaining Magicsock composition, physical roaming or platform acceptance boundaries. +The aligned candidate also combines the real Native encoder/forwarder/receiver +with the actual Browser bridge over the experimental Tailcat socket. Chromium +152 and Firefox 151 each pass an unchanged ordinary-Pion control and both +Tailcat address-change directions. After DERP closes, new blue content keeps +rendering on the same track. Firefox renegotiates its local leg once when that +endpoint moves; Chromium retains its working loopback leg. Local HTTP replaces +Native control only in this fixture; App authorization and the room controller +are not exercised. No product timeout or address policy changes for this check. + +The matching FFmpeg matrix passed five of six cases. One source-address handoff +kept decoded-frame counts advancing but lost the expected new colour; three +focused repetitions then passed. Decoder errors and this intermittent result +remain unresolved. The FFmpeg sink's direct SDK assembler differs from the +product relay-input owner, so the observation needs a matched bitstream/assembler +control before attributing it to product transport. Browser passes do not erase +that failed control or establish universal continuity. + Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof required fresh bidirectional delivery after DERP closed. One side reporting a diff --git a/docs/todo.md b/docs/todo.md index 595f08993..76918972e 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -79,8 +79,11 @@ Implementation needs a concrete useful result within the product model. exposed a separate local Browser bridge limit. Fresh interface gathering and bounded bridge renegotiation now preserve the local track in complete ordinary Pion App controls, including Firefox Host and Viewer. Retain the existing - address policy: forcing loopback fails Firefox. Verify the combined Magicsock - and Browser pipeline before claiming complete App recovery through that transport. + address policy: forcing loopback fails Firefox. Combined Native-engine/Browser + bridge controls pass both handoff directions in Chromium and Firefox; full App + authorization and room routing remain outside that fixture. Resolve the + intermittent FFmpeg decoded-content failure with a matched assembler/bitstream + control before claiming complete recovery through that transport. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) and RTCP/MTU boundaries, retained-flow recovery beyond the passing serial From 8867a1e9e8afddcf5f5b8869e7d9d2222c82344b Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 04:43:41 +0800 Subject: [PATCH 093/262] docs(research): reconcile decoded transport controls Record the matched assembler comparison and aligned six-case acceptance without claiming complete Magicsock product integration. Keep remaining work in TODO and completed evidence in its research owner. --- docs/research/nat-traversal.md | 19 ++++++++++++------- docs/todo.md | 19 +++++-------------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index fd7368fb0..9623ec70f 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -750,13 +750,18 @@ endpoint moves; Chromium retains its working loopback leg. Local HTTP replaces Native control only in this fixture; App authorization and the room controller are not exercised. No product timeout or address policy changes for this check. -The matching FFmpeg matrix passed five of six cases. One source-address handoff -kept decoded-frame counts advancing but lost the expected new colour; three -focused repetitions then passed. Decoder errors and this intermittent result -remain unresolved. The FFmpeg sink's direct SDK assembler differs from the -product relay-input owner, so the observation needs a matched bitstream/assembler -control before attributing it to product transport. Browser passes do not erase -that failed control or establish universal continuity. +The earlier FFmpeg matrix had one decoded-content failure. Matched synthetic +bitstream/RTP replay identifies a receiver-fixture mismatch: its separate SDK +assembler emits a truncated startup frame and can resume dependent frames after +a sequence gap without their reference. Increasing that builder's window instead +stalls recovery. On identical unchanged and source-handoff recordings, the +product relay-input owner's independent-keyframe boundary produces only +byte-matched source frames, with no decoder errors or corrupted colours. +The fixture now reuses that owner. All six unchanged, address-change and loss +controls pass with actual encoding/decoding and Linux race detection, without +decoder errors or changes to product timeout, adaptation or recovery policy. +The original failed run has no packet recording, so its exact cause remains +unassigned; this correction does not establish universal media continuity. Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof diff --git a/docs/todo.md b/docs/todo.md index 76918972e..a91eb987e 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -70,20 +70,11 @@ Implementation needs a concrete useful result within the product model. [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) have not established improved direct reachability. Seek a repeatable benefit before accepting room-scoped peer keys, flow admission and relay operation. - Isolated local-address changes now preserve the same session under the actual - Viewer recovery owner. Verify that bounded benefit through the complete App - pipeline and physical networks; it does not establish another reachable NAT class. - Live-encoder/decoded-picture controls now separate shared forwarding queue - delay from the local playback leg losing its selected LAN address. The shared - send-queue repair passes constrained-link recovery. The complete App controls - exposed a separate local Browser bridge limit. Fresh interface gathering and - bounded bridge renegotiation now preserve the local track in complete ordinary - Pion App controls, including Firefox Host and Viewer. Retain the existing - address policy: forcing loopback fails Firefox. Combined Native-engine/Browser - bridge controls pass both handoff directions in Chromium and Firefox; full App - authorization and room routing remain outside that fixture. Resolve the - intermittent FFmpeg decoded-content failure with a matched assembler/bitstream - control before claiming complete recovery through that transport. + Verify the measured local-address handoff benefit through the complete App, + mixed Browser/Native topology and physical networks. Current Native-engine, + Browser-bridge and decoded-picture controls do not cover App authorization or + room routing, and do not establish another reachable NAT class. Preserve the + existing local address policy; forcing loopback fails Firefox. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) and RTCP/MTU boundaries, retained-flow recovery beyond the passing serial From 20cb07de015f61653a6d2988765f783e6d7070ad Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 05:05:37 +0800 Subject: [PATCH 094/262] docs(research): record fresh-flow recovery alternative Keep the supported dependency, complete App and room authorization gates explicit. The isolated preface-free experiment narrows recovery composition without accepting a product transport or timeout change. --- docs/research/nat-traversal.md | 16 ++++++++++++++++ docs/todo.md | 7 ++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 9623ec70f..3950a02f0 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -607,6 +607,22 @@ retained-flow/64-check nine-/eleven-second, permanent-loss and retirement cases. These checks do not define room-grant integration or establish full App interoperability or any additional direct-reachability benefit. +A preface-free alternative uses Tailcat's existing +[UDP listener](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/listen.go) +to reserve the address before SDP, allowing ICE to supply the first authenticated +datagram. Each replacement owns a fresh flow; no old mux survives it. An +unaccepted write returns a socket error instead of blocking ICE's event loop. +In the combined dependency candidate, 64-check nine-/eleven-second controls +each pass two Linux race repetitions under the actual Viewer owner. Setup takes +12–15 ms; fresh media returns at about 14.2 seconds from blackout onset, within +the unchanged two-request and initial-connection budgets. Permanent loss still +exhausts recovery, and retirement closes pending work. Local race controls cover +mutable deadlines, closing during ICE, peer rejection and continuous sibling +audio/video. Default inner checks still fail in the matched outage controls. +This removes the fixture-preface and retained-flow lifetime requirements; it +does not accept a product Binding limit, implement room grants or establish +full App/decoded-media acceptance of this alternative. + Separate checks preserve continuous sibling audio/video through a local ICE restart, failed-flow retirement, both callers' pending cancellation/deadline, revocation during a Server probe and unknown-peer rejection. Canceling a probe diff --git a/docs/todo.md b/docs/todo.md index a91eb987e..f1891a97a 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -77,9 +77,10 @@ Implementation needs a concrete useful result within the product model. existing local address policy; forcing loopback fails Firefox. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) - and RTCP/MTU boundaries, retained-flow recovery beyond the passing serial - Trickle/64-check experiment, and mixed Browser/Native verification. Default - inner checks still fail; no new product setting or flow owner is accepted. + and RTCP/MTU boundaries, complete recovery with fresh per-connection flows + beyond the isolated Trickle/64-check experiment, and mixed Browser/Native + verification. Default inner checks still fail; no new product setting or flow + owner is accepted. Keep isolated dependency patches and recovery counterfactuals outside product routing; a diagnostic reply is not media-readiness authority. Research owns the experiments and their limits. From 4735acd31762282947f9327709a5825ea5943e91 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 05:24:52 +0800 Subject: [PATCH 095/262] docs(transport): distinguish native recovery branches --- docs/research/nat-traversal.md | 57 ++++++++++++++++------------- docs/standards/routing-transport.md | 6 ++- docs/todo.md | 8 ++-- 3 files changed, 41 insertions(+), 30 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 3950a02f0..32cda4b0a 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -590,22 +590,14 @@ A retained-flow counterfactual reuses Pion's existing [UDP mux](https://github.com/pion/ice/blob/v4.4.0/udp_mux.go) after closing the old PeerConnection. It keeps the accepted Tailcat flow and replaces ICE/DTLS/SRTP, without another listener or preparation exchange. Setup falls from about 913 ms -to 11–20 ms, but only three of six default-check nine-/eleven-second controls -recover. The other three exhaust inner checks without fresh SRTP. This removes -flow creation from those failures; it does not by itself resolve recovery. -Combining retained flow with 16 checks still fails two of six cases. With 64 -checks, all six nine-/eleven-second cases recover; permanent loss still exhausts -the existing two-request budget after the replacement's 15-second initial -deadline, and disposal cancels recovery. This identifies a bounded composition -candidate without another timer or readiness probe. Each retained flow is serial: -overlapping connections still use separate flows. Three local race controls keep -sibling audio/video flowing through two retained-flow replacements and another -flow's rollback; peer revocation stops fresh media on both remaining flows. -The combined listener-state, Tailcat/WireGuard revocation and bounded-TWCC -candidates also pass Linux race checks for those isolation boundaries and the -retained-flow/64-check nine-/eleven-second, permanent-loss and retirement cases. -These checks do not define room-grant integration or establish full App -interoperability or any additional direct-reachability benefit. +to 11–20 ms. Nine-/eleven-second recovery passes three of six default-check +cases, four of six with 16 checks and six of six with 64. Removing flow creation +alone therefore does not resolve recovery. Combined listener/revocation/TWCC +patches pass Linux race checks for 64-check recovery, bounded exhaustion, +retirement and sibling audio/video through serial replacements and rollback. +Overlapping connections still use separate flows, and revocation stops both. +This counterfactual does not accept retained flow as a product owner; the fresh +flow alternative below avoids that extra lifetime. A preface-free alternative uses Tailcat's existing [UDP listener](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/listen.go) @@ -618,10 +610,22 @@ each pass two Linux race repetitions under the actual Viewer owner. Setup takes the unchanged two-request and initial-connection budgets. Permanent loss still exhausts recovery, and retirement closes pending work. Local race controls cover mutable deadlines, closing during ICE, peer rejection and continuous sibling -audio/video. Default inner checks still fail in the matched outage controls. +audio/video. Default inner checks still fail in these restart-then-rebuild controls. This removes the fixture-preface and retained-flow lifetime requirements; it does not accept a product Binding limit, implement room grants or establish -full App/decoded-media acceptance of this alternative. +full App recovery acceptance of this alternative. + +The sender's [restart capability](../standards/routing-transport.md#reconciliation) +changes that schedule. Modeling the current Native sender's first-request +replacement under the same Viewer owner on 2026-10-10, default checks recover both +nine- and eleven-second outages at about 23.3 seconds after two replacements. Their first +replacement still exhausts eight checks; the second starts after its existing +15-second initial deadline. With 64 checks, these controls recover after one +replacement at about 12.2 and 15.3 seconds respectively. Permanent loss exhausts +two replacements at about 38.3 seconds, and disposal cancels pending recovery. +This narrows the earlier failure claim to its tested sender behavior; it is not a universal Native failure +or an accepted product tuning. Native control and decoding remain stubbed in +this owner fixture, distinct from the decoded handoff checks below. Separate checks preserve continuous sibling audio/video through a local ICE restart, failed-flow retirement, both callers' pending cancellation/deadline, @@ -758,13 +762,16 @@ bridge gap, not the remaining Magicsock composition, physical roaming or platfor acceptance boundaries. The aligned candidate also combines the real Native encoder/forwarder/receiver -with the actual Browser bridge over the experimental Tailcat socket. Chromium -152 and Firefox 151 each pass an unchanged ordinary-Pion control and both -Tailcat address-change directions. After DERP closes, new blue content keeps -rendering on the same track. Firefox renegotiates its local leg once when that -endpoint moves; Chromium retains its working loopback leg. Local HTTP replaces -Native control only in this fixture; App authorization and the room controller -are not exercised. No product timeout or address policy changes for this check. +with the actual Browser bridge over the experimental Tailcat socket. The fresh +flow adapter now replaces its synthetic preface: SDP starts before peer admission, +and ICE supplies the first datagram. All six FFmpeg-decoded controls pass Linux +race checks. Chromium 152 and Firefox 151 each pass an unchanged ordinary-Pion +control and both Tailcat address-change directions. After DERP closes, new blue +content keeps rendering on the same track. Firefox renegotiates its local leg +once when that endpoint moves; Chromium retains its working loopback leg. Local +HTTP replaces Native control only in this fixture; App authorization, room routing +and prolonged-outage replacement are not exercised. No product timeout, address +policy or inner Binding setting changes for these handoff checks. The earlier FFmpeg matrix had one decoded-content failure. Matched synthetic bitstream/RTP replay identifies a receiver-fixture mismatch: its separate SDK diff --git a/docs/standards/routing-transport.md b/docs/standards/routing-transport.md index 4397c54b9..1044a7af3 100644 --- a/docs/standards/routing-transport.md +++ b/docs/standards/routing-transport.md @@ -119,8 +119,10 @@ deadline or start an independent recovery loop. When NAT traversal enhancement is enabled, another operation may use the remaining connection-attempt budget after rollback. The waiting display uses the actual server-issued attempt ordinal; it never counts time as an attempted connection. -Committed P2P recovery first restarts ICE on the same connection, then -rebuilds that connection if necessary, within the existing two-request budget. +Committed P2P recovery first requests an ICE restart, then requests a rebuild +if necessary, within the existing two-request budget. A sender without restart +support, including the current Native sender, replaces its connection on that +first request. The budget belongs to the current parent and survives physical connection or Browser/Native backend replacement. Only a connected transport or a new parent resets it; replacing an unconnected backend cannot renew recovery indefinitely. diff --git a/docs/todo.md b/docs/todo.md index f1891a97a..5838625e6 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -78,9 +78,11 @@ Implementation needs a concrete useful result within the product model. Product adoption still needs supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) and RTCP/MTU boundaries, complete recovery with fresh per-connection flows - beyond the isolated Trickle/64-check experiment, and mixed Browser/Native - verification. Default inner checks still fail; no new product setting or flow - owner is accepted. + beyond the isolated owner and decoded-handoff checks, including each sender's + recovery capability, and mixed Browser/Native verification. Inner checks can + exhaust before the underlay recovers; research distinguishes eventual recovery + from exhaustion by sender behavior. No new product setting or flow owner is + accepted. Keep isolated dependency patches and recovery counterfactuals outside product routing; a diagnostic reply is not media-readiness authority. Research owns the experiments and their limits. From e780440c6e38204bc31d7fecc01d437710fc2f12 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 05:42:33 +0800 Subject: [PATCH 096/262] docs(research): record native admission ordering constraint --- docs/research/nat-traversal.md | 20 ++++++++++++++++++++ docs/todo.md | 2 ++ 2 files changed, 22 insertions(+) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 32cda4b0a..4c5a5527c 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -773,6 +773,15 @@ HTTP replaces Native control only in this fixture; App authorization, room routi and prolonged-outage replacement are not exercised. No product timeout, address policy or inner Binding setting changes for these handoff checks. +A separate Linux race check now passes those experimental flows through real +Native control commands and the Viewer session's existing receiver owner. +Closing a receiver releases its physical flow; replacement and a healthy sibling +keep receiving fresh RTP after DERP closes, including after late cleanup for the +old connection ID. Stopping reception or closing the control session releases +both remaining flows. Omitting receiver retirement in the test overlay is +detected. Keys are still pre-granted locally: this establishes resource cleanup, +not room authorization, automatic App recovery or Browser decoding in this check. + The earlier FFmpeg matrix had one decoded-content failure. Matched synthetic bitstream/RTP replay identifies a receiver-fixture mismatch: its separate SDK assembler emits a truncated startup frame and can resume dependent frames after @@ -864,6 +873,17 @@ NAT, dual-stack, rebinding and weak-network controls also pass. Supported depend contracts and demonstrated network benefit remain open. The probe retains the isolated patches; Piik's production transport is unchanged. +Admission ordering is also an integration constraint. Upstream +[`Client.DialUDPPort`](https://github.com/tailscale/tailcat/blob/52fbad323e9d352f1febb623e0975368e9a8cbdd/tailcat.go) +waits for the server to admit the client's key before returning a socket. +Piik creates the receiver and its socket before returning a media answer. +Authorizing that key only after receiving the answer would therefore introduce +a circular wait. Existing fixtures pre-grant keys and do not test this exchange. +A product design must resolve that ordering within the existing assigned +connection's authority; accepting arbitrary keys or passing room credentials +into Native control is not an equivalent shortcut. This is a source-traced +integration constraint, not a defect in Piik's current transport. + ## Native Mapping Cache And Retry Lifetimes Pion ICE `v4.4.0` caches a shared socket's STUN mapping by destination for 25 diff --git a/docs/todo.md b/docs/todo.md index 5838625e6..838fc799e 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -83,6 +83,8 @@ Implementation needs a concrete useful result within the product model. exhaust before the underlay recovers; research distinguishes eventual recovery from exhaustion by sender behavior. No new product setting or flow owner is accepted. + Resolve admission-before-answer ordering at the assigned-connection owner; + research owns the Native control retirement checks and their pre-granted scope. Keep isolated dependency patches and recovery counterfactuals outside product routing; a diagnostic reply is not media-readiness authority. Research owns the experiments and their limits. From 1ba6021e0a2e8cc14c952881299fa52d817cb9db Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 06:05:00 +0800 Subject: [PATCH 097/262] fix(app): bound native viewer upstream receivers --- internal/app/nativeviewer/session.go | 12 +++++- internal/app/nativeviewer/session_test.go | 48 +++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/internal/app/nativeviewer/session.go b/internal/app/nativeviewer/session.go index a2aa88cb7..4ee448813 100644 --- a/internal/app/nativeviewer/session.go +++ b/internal/app/nativeviewer/session.go @@ -37,6 +37,9 @@ type sessionEdge struct { sourceConnectionID string } +// One current upstream and one prepared route may overlap during a handoff. +const receiverCapacity = 2 + // Session owns the native media implementation for one Browser Viewer. Room, // participant, route, and recovery authority remain in the Browser/server. type Session struct { @@ -123,11 +126,16 @@ func (session *Session) AcceptOffer( } session.mu.Lock() closed := session.closed + receiver := session.receivers[connectionID] + atCapacity := len(session.receivers) >= receiverCapacity && receiver == nil session.mu.Unlock() if closed { return OfferResult{}, errors.New("native Viewer session is closed") } - if receiver := session.receiver(connectionID); reuse && receiver != nil { + if atCapacity { + return OfferResult{}, errors.New("native Viewer receiver capacity reached") + } + if reuse && receiver != nil { answer, reused, err := receiver.Renegotiate(offer, iceServers) if err != nil { return OfferResult{}, err @@ -173,7 +181,7 @@ func (session *Session) AcceptOffer( return OfferResult{}, err } session.mu.Lock() - if session.closed || session.receivers[connectionID] != nil { + if session.closed || session.receivers[connectionID] != nil || len(session.receivers) >= receiverCapacity { session.mu.Unlock() _ = receiver.Close() return OfferResult{}, errors.New("native Viewer receiver is unavailable") diff --git a/internal/app/nativeviewer/session_test.go b/internal/app/nativeviewer/session_test.go index 5123c12b5..3ce6837c9 100644 --- a/internal/app/nativeviewer/session_test.go +++ b/internal/app/nativeviewer/session_test.go @@ -86,3 +86,51 @@ func TestViewerProfileHasNoImplicitCaptureDefault(t *testing.T) { t.Fatal("closed Viewer accepted a profile") } } + +func TestReceiverCapacityPreservesCurrentAndPreparedSources(t *testing.T) { + peer, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = peer.Close() }) + if _, err = peer.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo); err != nil { + t.Fatal(err) + } + offer, err := peer.CreateOffer(nil) + if err != nil { + t.Fatal(err) + } + session, err := Start(t.Context(), Options{ShareID: "viewer-session", EdgeCapacity: 2}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = session.Close() }) + for _, id := range []string{"active", "candidate"} { + if _, err = session.AcceptOffer(id, offer, nil, false); err != nil { + t.Fatal(err) + } + } + active, candidate := session.receiver("active"), session.receiver("candidate") + if _, err = session.AcceptOffer("excess", offer, nil, false); err == nil { + t.Fatal("a third receiver bypassed the active/candidate bound") + } + if session.receiver("active") != active || session.receiver("candidate") != candidate || session.receiver("excess") != nil { + t.Fatal("capacity rejection changed an existing receiver") + } + if result, err := session.AcceptOffer("active", offer, nil, true); err != nil || !result.Reused { + t.Fatalf("existing receiver could not renegotiate at capacity: %v", err) + } + if _, err = session.PrepareLocalEdge("active", "local-preview", "preview-offer", false); err != nil { + t.Fatalf("capacity rejection stopped local playback preparation: %v", err) + } + if _, err = session.AcceptOffer("candidate", offer, nil, false); err != nil { + t.Fatalf("existing receiver could not be replaced at capacity: %v", err) + } + if session.receiver("active") != active || session.receiver("candidate") == candidate || session.edge("local-preview") == nil { + t.Fatal("receiver replacement changed the active source or its playback") + } + session.CloseReceiver("candidate") + if _, err = session.AcceptOffer("replacement", offer, nil, false); err != nil { + t.Fatalf("retired receiver did not release its slot: %v", err) + } +} From 18144a7665e246dad0b604e646980a7a4528f57b Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 06:19:21 +0800 Subject: [PATCH 098/262] fix(media): initialize source state before feedback starts --- docs/standards/engineering.md | 2 ++ internal/app/mediaedge/edge_test.go | 6 ++-- internal/app/mediaedge/engine.go | 15 +++++++-- internal/app/mediaedge/receiver.go | 7 ++--- internal/app/mediaedge/relay_fixture_test.go | 18 +++++------ .../mediaedge/relay_network_fixture_test.go | 9 +++--- internal/app/mediaedge/source_test.go | 31 +++++++++++++++++++ 7 files changed, 63 insertions(+), 25 deletions(-) diff --git a/docs/standards/engineering.md b/docs/standards/engineering.md index 048a4bc80..f59b0501f 100644 --- a/docs/standards/engineering.md +++ b/docs/standards/engineering.md @@ -171,6 +171,8 @@ above rather than searching the tree. Construction is part of acquisition: if it throws, release resources created before the failure and report through the existing operation owner. A failed allocation does not erase the operation's authoritative identity. + Initialize callback-visible state before registration or binding; library + constructors can start background work before returning. Ordered shutdown may require a ready auxiliary transport to outlive the stop request; its owner closes it after dependants retire. External convenience actions must not block lifecycle handling, and repeated attempts stay bounded. diff --git a/internal/app/mediaedge/edge_test.go b/internal/app/mediaedge/edge_test.go index a4e78c2a4..f251d31be 100644 --- a/internal/app/mediaedge/edge_test.go +++ b/internal/app/mediaedge/edge_test.go @@ -125,12 +125,11 @@ func TestRelayFailureCannotRetireReconfiguredProfile(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - source, err := engine.NewSource("vp8", 1, 2, nil) + source, err := engine.newSource("vp8", 1, 2, &RelayOptions{}, nil) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = source.Close() }) - source.relay = &relayDerivation{source: source} _ = source.SetFormat(1, 640, 360) profile := nativecapture.VideoProfile{Width: 1280, Height: 720, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"} if err = source.SetRelayProfile(profile); err != nil { @@ -182,12 +181,11 @@ func TestRetiredRelayFailureCannotDisableLaterDemand(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - source, err := engine.NewSource("vp8", 2, 2, nil) + source, err := engine.newSource("vp8", 2, 2, &RelayOptions{}, nil) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = source.Close() }) - source.relay = &relayDerivation{source: source} if err = source.SetFormat(1, 640, 360); err != nil { t.Fatal(err) } diff --git a/internal/app/mediaedge/engine.go b/internal/app/mediaedge/engine.go index 3f444b52c..351494c96 100644 --- a/internal/app/mediaedge/engine.go +++ b/internal/app/mediaedge/engine.go @@ -137,6 +137,10 @@ func stunServers(servers []webrtc.ICEServer) []webrtc.ICEServer { // NewSource coalesces recovery requests by physical output slot; [-1] requests // all outputs. The callback runs outside the source lock and forwarding path. func (engine *Engine) NewSource(codec string, capacity, layers int, requestKeyFrame func([]int)) (*Source, error) { + return engine.newSource(codec, capacity, layers, nil, requestKeyFrame) +} + +func (engine *Engine) newSource(codec string, capacity, layers int, relay *RelayOptions, requestKeyFrame func([]int)) (*Source, error) { _, supported := videoCodecs[codec] if !supported { return nil, errors.New("native video codec is unsupported") @@ -157,6 +161,15 @@ func (engine *Engine) NewSource(codec string, capacity, layers int, requestKeyFr capacity: capacity, requestKeyFrame: requestKeyFrame, edges: make(map[*Edge]bool), publications: make(map[*Publication]struct{}), } + // Binding forwarding buffers can start asynchronous PLI callbacks. Publish + // their immutable inputs before binding, and start our worker only on success. + if requestKeyFrame != nil { + source.recoveryRequests = make(chan struct{}, 1) + source.recoveryStopped = make(chan struct{}) + } + if relay != nil { + source.relay = &relayDerivation{source: source, options: *relay} + } media, err := forwarding.NewEncodedSource(forwarding.SourceOptions{ ID: "screen", StreamID: "piik-native", Codec: videoCodecs[codec], Formats: make([]forwarding.LayerFormat, layers), MaxPackets: encoded.MaxPacketWindow, @@ -174,8 +187,6 @@ func (engine *Engine) NewSource(codec string, capacity, layers int, requestKeyFr } source.media = media if requestKeyFrame != nil { - source.recoveryRequests = make(chan struct{}, 1) - source.recoveryStopped = make(chan struct{}) go source.runRecovery() } return source, nil diff --git a/internal/app/mediaedge/receiver.go b/internal/app/mediaedge/receiver.go index 4ca2a18f9..e9d01a30b 100644 --- a/internal/app/mediaedge/receiver.go +++ b/internal/app/mediaedge/receiver.go @@ -92,10 +92,12 @@ func (engine *Engine) NewReceiver(options ReceiverOptions) (_ *Receiver, _ webrt prepareMapping: prepareMapping, } layers := 1 + var relay *RelayOptions if _, supported := relayBackend(videoCodec, options.Relay); supported { layers = 2 + relay = options.Relay } - receiver.source, err = engine.NewSource(videoCodec, options.EdgeCapacity, layers, func([]int) { + receiver.source, err = engine.newSource(videoCodec, options.EdgeCapacity, layers, relay, func([]int) { // Local output slots all derive from this one upstream encoded stream. receiver.RequestKeyFrame() }) @@ -103,9 +105,6 @@ func (engine *Engine) NewReceiver(options ReceiverOptions) (_ *Receiver, _ webrt _ = connection.Close() return nil, webrtc.SessionDescription{}, err } - if layers > 1 { - receiver.source.relay = &relayDerivation{source: receiver.source, options: *options.Relay} - } if hasAudio { receiver.audioSource, err = engine.NewRelayedAudioSource(options.EdgeCapacity) if err != nil { diff --git a/internal/app/mediaedge/relay_fixture_test.go b/internal/app/mediaedge/relay_fixture_test.go index 79f050b43..3add2670d 100644 --- a/internal/app/mediaedge/relay_fixture_test.go +++ b/internal/app/mediaedge/relay_fixture_test.go @@ -51,15 +51,14 @@ func TestRelayDerivationFixture(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - source, err := engine.NewSource("vp8", 2, 2, nil) + source, err := engine.newSource("vp8", 2, 2, &RelayOptions{ + CaptureProcess: executable, + Capabilities: nativecapture.Capabilities{SoftwareVP8: true, Adapters: []nativecapture.Adapter{{Index: 0}}}, + }, nil) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = source.Close() }) - source.relay = &relayDerivation{source: source, options: RelayOptions{ - CaptureProcess: executable, - Capabilities: nativecapture.Capabilities{SoftwareVP8: true, Adapters: []nativecapture.Adapter{{Index: 0}}}, - }} profile := nativecapture.VideoProfile{Width: 1280, Height: 720, Framerate: 30, Bitrate: 2_000_000, Preference: "balanced"} if err = source.SetRelayProfile(profile); err != nil { t.Fatal(err) @@ -330,13 +329,12 @@ func TestRelayIndependentSpatialFixture(t *testing.T) { engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) check(err) defer engine.Close() - source, err := engine.NewSource("vp8", 2, 2, nil) - check(err) - defer source.Close() - source.relay = &relayDerivation{source: source, options: RelayOptions{ + source, err := engine.newSource("vp8", 2, 2, &RelayOptions{ CaptureProcess: executable, Capabilities: nativecapture.Capabilities{SoftwareVP8: true, Adapters: []nativecapture.Adapter{{Index: 0}}}, - }} + }, nil) + check(err) + defer source.Close() check(source.SetRelayProfile(nativecapture.VideoProfile{ Width: 1920, Height: 1080, Framerate: 30, Bitrate: 5_000_000, Preference: "maintain-framerate", })) diff --git a/internal/app/mediaedge/relay_network_fixture_test.go b/internal/app/mediaedge/relay_network_fixture_test.go index ea8c1d550..8f1958b0e 100644 --- a/internal/app/mediaedge/relay_network_fixture_test.go +++ b/internal/app/mediaedge/relay_network_fixture_test.go @@ -66,13 +66,12 @@ func TestRelayNetworkVP8Fixture(t *testing.T) { engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", InitialBitrate: 1_200_000}) check(err) defer engine.Close() - source, err := engine.NewSource("vp8", 2, 2, nil) - check(err) - defer source.Close() - source.relay = &relayDerivation{source: source, options: RelayOptions{ + source, err := engine.newSource("vp8", 2, 2, &RelayOptions{ CaptureProcess: os.Getenv("PIIK_NATIVE_CAPTURE"), Capabilities: nativecapture.Capabilities{SoftwareVP8: true, Adapters: []nativecapture.Adapter{{Index: 0}}}, - }} + }, nil) + check(err) + defer source.Close() check(source.SetRelayProfile(nativecapture.VideoProfile{Width: 1280, Height: 720, Framerate: 30, Bitrate: 2_000_000, Preference: "balanced"})) router, err := vnet.NewRouter(&vnet.RouterConfig{CIDR: "10.0.0.0/24", MinDelay: 20 * time.Millisecond, diff --git a/internal/app/mediaedge/source_test.go b/internal/app/mediaedge/source_test.go index 7d70a28d4..c9687a788 100644 --- a/internal/app/mediaedge/source_test.go +++ b/internal/app/mediaedge/source_test.go @@ -56,6 +56,37 @@ func TestSourceRecoveryCoalescesTargetsWithoutWakingUnrelatedOutputs(t *testing. } } +func TestSourceStartupFeedbackRequestsRecovery(t *testing.T) { + for _, relay := range []*RelayOptions{nil, {}} { + t.Run(fmt.Sprintf("relay-%t", relay != nil), func(t *testing.T) { + requested := make(chan []int, 4) + source, err := (&Engine{}).newSource("vp8", 2, 2, relay, func(layers []int) { + select { + case requested <- layers: + default: + } + }) + if err != nil { + t.Fatal(err) + } + defer source.Close() + // LiveKit can request startup keyframes as soon as its buffers are bound. + // Wait for that real asynchronous callback, without manually sending PLI. + select { + case layers := <-requested: + if len(layers) == 0 { + t.Fatal("startup recovery has no output identity") + } + if (source.groupRecovery.Load() != 0) != (relay != nil) { + t.Fatal("startup recovery used the wrong capture/relay owner") + } + case <-time.After(3 * time.Second): + t.Fatal("startup feedback lost its recovery notification") + } + }) + } +} + func TestSourceRecoveryRetainsLocalPlanWithoutDirectCaptureCallback(t *testing.T) { for _, relay := range []bool{false, true} { t.Run(fmt.Sprint(relay), func(t *testing.T) { From e1209e3e25bf457ee4ef7dd71922d59bf0bd30e6 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 06:19:48 +0800 Subject: [PATCH 099/262] docs(research): qualify native admission ordering evidence --- docs/research/nat-traversal.md | 21 ++++++++++++++++----- docs/todo.md | 9 +++++++-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 4c5a5527c..7ddcdb0f1 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -878,11 +878,22 @@ Admission ordering is also an integration constraint. Upstream waits for the server to admit the client's key before returning a socket. Piik creates the receiver and its socket before returning a media answer. Authorizing that key only after receiving the answer would therefore introduce -a circular wait. Existing fixtures pre-grant keys and do not test this exchange. -A product design must resolve that ordering within the existing assigned -connection's authority; accepting arbitrary keys or passing room credentials -into Native control is not an equivalent shortcut. This is a source-traced -integration constraint, not a defect in Piik's current transport. +a circular wait. An isolated `PrepareUDPPort` patch reserves the local address +without waiting for admission. Through real Native control, it can return the +answer first and receive fresh RTP if the fixture holds remote ICE candidates +until the grant completes. Both receiver/control retirement paths retain their +cleanup checks; the pre-granted control uses the same client/server orientation. +Without that experimental candidate gate, Native sends five ICE checks before +the roughly one-second grant acknowledgment, reaches eight and receives no RTP +within the unchanged bound. Complete-SDP and early-trickle controls both fail. +The early-trickle counterfactual is opt-in and intentionally fails its media check. + +This establishes a startup-order constraint, not a product integration or a Piik +field defect. Keys and grants remain local fixture inputs; the candidate gate +and unsupported dependency method are not accepted product mechanisms. +The existing assigned-connection owner must resolve key exchange and admission +before ICE starts. Accepting arbitrary keys, passing room credentials into Native +control or enlarging its timeout is not an equivalent solution. ## Native Mapping Cache And Retry Lifetimes diff --git a/docs/todo.md b/docs/todo.md index 838fc799e..ad7ee5107 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -16,6 +16,10 @@ Git/PRs own completed history. Prepare one coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. + Isolate the intermittent Linux race-suite no-output result in + `TestQualityDimensionsFollowAssignedSource`; focused repetitions and the next + full package run pass. No product cause is established; preserve its assertion + and timeout while collecting a matched failure. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance are complete. Integrate through the next owner-approved publishing merge and verify the separate workflow with its original CI artifacts. Mirror failure @@ -83,8 +87,9 @@ Implementation needs a concrete useful result within the product model. exhaust before the underlay recovers; research distinguishes eventual recovery from exhaustion by sender behavior. No new product setting or flow owner is accepted. - Resolve admission-before-answer ordering at the assigned-connection owner; - research owns the Native control retirement checks and their pre-granted scope. + Resolve key exchange and admission before ICE at the assigned-connection owner; + research owns the Native control retirement checks and the isolated candidate + gate. Neither the pre-granted nor gated fixture implements room authorization. Keep isolated dependency patches and recovery counterfactuals outside product routing; a diagnostic reply is not media-readiness authority. Research owns the experiments and their limits. From 9067d8c18ad004411f6426e25858dff36fc2f84c Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 06:46:26 +0800 Subject: [PATCH 100/262] docs(research): record cross-browser H264 receiving limits --- docs/research/native-client-media.md | 17 +++++++++++++++-- docs/todo.md | 3 +++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 2969d94af..77e1047b3 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -644,11 +644,24 @@ old SFU picture during peer preparation. The mixed Browser Host/Native Viewer control also passed those transitions. With clarity priority, its live 2160x3840-to-360x640 edit retained the Native connection and playback track; SFU recovery retained that size and then restored 2160x3840. Firefox 155 received -the same VP8 sizes from Chrome, retaining its track through both edits. All -controls retired their receiving connections on stop. Balanced priority may +the same VP8 and H.264 sizes from Chrome 154, retaining its track through both +edits. All controls retired their receiving connections on stop. Balanced priority may deliver smaller adapted pictures. These local checks do not establish physical 4K capture, every mixed handoff or cross-network performance. +The H.264 control supplies OpenH264 2.6.0 to an isolated Firefox profile, using +the artifact and SHA-512 from [Mozilla's codec manifest](https://github.com/mozilla-firefox/firefox/blob/main/toolkit/content/gmp-sources/openh264.json). +This follows the [Web Platform Tests setup](https://github.com/web-platform-tests/wpt/blob/master/tools/wpt/browser.py). +A fresh-profile control did not advertise H.264 and could not receive the +H.264-only share, so codec availability must be established before attributing +that result to resolution or networking. Firefox owns its component installation; +the [upstream report](https://bugzilla.mozilla.org/show_bug.cgi?id=1766638) records +the separate first-use installation boundary. +A controlled receiver with H.264 unavailable reaches the current generic +“No media route available” presentation. Receiving feedback does not yet retain +that confirmed incompatibility through route exhaustion; capability work needs +to distinguish it without introducing another retry or failure owner. + Chrome 154 camera controls exposed a separate constraint bug: lowering a 540x960 input to the 1280x720 ceiling produced a cropped 540x720 picture when both axes also carried `ideal` values. Maximum-only constraints produced diff --git a/docs/todo.md b/docs/todo.md index ad7ee5107..05919cf1e 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -55,6 +55,9 @@ Implementation needs a concrete useful result within the product model. owns local capture, codec, derived-output, SFU and bounded mixed-route evidence. Extend Browser interoperability and mixed-topology checks to limited receivers and physical camera orientation behavior. + Preserve confirmed decoder incompatibility in receiving feedback: the local + H.264-unavailable control currently ends with generic route exhaustion. + Keep the existing presentation and recovery owners. Browser display capture, Browser relay and Browser-to-Native fanout now have bounded high-resolution evidence. macOS geometry and physical 4K/game capture remain open. Preserve safety From ae9014b42af2b1d5c71c7082564d5e2f3a352ec9 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 07:00:13 +0800 Subject: [PATCH 101/262] fix(viewer): preserve negotiated video rejection --- docs/research/native-client-media.md | 16 ++++++---- docs/todo.md | 3 -- src/client/locales/en.ts | 1 + src/client/locales/zh.ts | 1 + src/client/media/viewer-presentation.ts | 15 ++++++++++ src/client/media/viewer-sfu-route.ts | 13 ++++---- src/client/native/media-bridge.ts | 2 ++ src/client/pages/ViewerPage.tsx | 13 ++++++++ src/client/pages/status-preview-scenarios.ts | 6 ++++ src/client/sfu/subscriber.ts | 17 ++++++----- src/client/ui/media-status.ts | 1 + src/client/webrtc/video-codec.ts | 24 +++++++++++++++ src/client/webrtc/viewer-peer.ts | 6 +++- tests/native-media-bridge.test.ts | 11 +++++++ tests/route-transition.test.ts | 8 +++++ tests/sfu-client.test.ts | 14 +++++++++ tests/video-codec.test.ts | 20 ++++++++++++- tests/viewer-peer-generation.test.ts | 22 +++++++++++++- tests/viewer-presentation.test.ts | 31 ++++++++++++++++++++ 19 files changed, 199 insertions(+), 25 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 77e1047b3..da70412a5 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -645,8 +645,9 @@ control also passed those transitions. With clarity priority, its live 2160x3840-to-360x640 edit retained the Native connection and playback track; SFU recovery retained that size and then restored 2160x3840. Firefox 155 received the same VP8 and H.264 sizes from Chrome 154, retaining its track through both -edits. All controls retired their receiving connections on stop. Balanced priority may -deliver smaller adapted pictures. These local checks do not establish physical +edits; H.264 also passed through embedded SFU. All controls retired their +receiving connections on stop. Balanced priority may deliver smaller adapted pictures. +These local checks do not establish physical 4K capture, every mixed handoff or cross-network performance. The H.264 control supplies OpenH264 2.6.0 to an isolated Firefox profile, using @@ -657,10 +658,13 @@ H.264-only share, so codec availability must be established before attributing that result to resolution or networking. Firefox owns its component installation; the [upstream report](https://bugzilla.mozilla.org/show_bug.cgi?id=1766638) records the separate first-use installation boundary. -A controlled receiver with H.264 unavailable reaches the current generic -“No media route available” presentation. Receiving feedback does not yet retain -that confirmed incompatibility through route exhaustion; capability work needs -to distinguish it without introducing another retry or failure owner. +With H.264 unavailable, the Browser accepts the offer but rejects its video +section in the answer. The shared receiving check uses this +[JSEP rejection](https://www.rfc-editor.org/rfc/rfc8829.html#section-5.3.1), rather +than inferring support from a codec list. P2P and SFU controls now retain that +cause through route exhaustion instead of reporting only a media-route failure; +the Native local bridge uses the same check and its existing failure path. +An answer rejection alone does not identify the decoder or platform cause. Chrome 154 camera controls exposed a separate constraint bug: lowering a 540x960 input to the 1280x720 ceiling produced a cropped 540x720 picture when diff --git a/docs/todo.md b/docs/todo.md index 05919cf1e..ad7ee5107 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -55,9 +55,6 @@ Implementation needs a concrete useful result within the product model. owns local capture, codec, derived-output, SFU and bounded mixed-route evidence. Extend Browser interoperability and mixed-topology checks to limited receivers and physical camera orientation behavior. - Preserve confirmed decoder incompatibility in receiving feedback: the local - H.264-unavailable control currently ends with generic route exhaustion. - Keep the existing presentation and recovery owners. Browser display capture, Browser relay and Browser-to-Native fanout now have bounded high-resolution evidence. macOS geometry and physical 4K/game capture remain open. Preserve safety diff --git a/src/client/locales/en.ts b/src/client/locales/en.ts index 95f3a59fd..161c5fb86 100644 --- a/src/client/locales/en.ts +++ b/src/client/locales/en.ts @@ -464,6 +464,7 @@ export const en: Record = { "viewer.msg.hostPaused": "Host paused", "viewer.msg.recovering": "Recovering connection", "viewer.msg.routeFailed": "No media route available; reload to retry", + "viewer.msg.videoRejected": "This browser cannot receive the video; try another browser", "viewer.msg.playbackFailed": "Cannot play this stream; reload to retry", "viewer.msg.hostOffline": "Host connection lost", "viewer.msg.serverError": "Connection service unavailable", diff --git a/src/client/locales/zh.ts b/src/client/locales/zh.ts index f17f1bbc4..2e6e02930 100644 --- a/src/client/locales/zh.ts +++ b/src/client/locales/zh.ts @@ -464,6 +464,7 @@ export const zh = { "viewer.msg.hostPaused": "房主已暂停分享", "viewer.msg.recovering": "正在恢复连接", "viewer.msg.routeFailed": "没有可用的媒体线路,可刷新重试", + "viewer.msg.videoRejected": "当前浏览器无法接收画面,请尝试更换浏览器", "viewer.msg.playbackFailed": "无法播放当前画面,请刷新重试", "viewer.msg.hostOffline": "房主连接已中断", "viewer.msg.serverError": "连接服务暂时不可用", diff --git a/src/client/media/viewer-presentation.ts b/src/client/media/viewer-presentation.ts index a833369f9..535770944 100644 --- a/src/client/media/viewer-presentation.ts +++ b/src/client/media/viewer-presentation.ts @@ -25,6 +25,7 @@ export type ViewerFailureCode = | "HOST_OFFLINE" | "ROUTE_EXHAUSTED" | "AUTOPLAY_BLOCKED" + | "VIDEO_REJECTED" | "PLAYBACK_FAILED"; export type ViewerMessageKey = Extract; @@ -46,6 +47,7 @@ export type ViewerStage = | "playing" | "recovering" | "route-failed" + | "video-rejected" | "playback-failed" | "waiting-host" | "host-offline" @@ -92,6 +94,7 @@ export interface ViewerPresentationState { autoplayBlockedGeneration: number | null; playbackFailedGeneration: number | null; runtimeFailure: "SERVER_ERROR" | null; + receiveFailure: "VIDEO_REJECTED" | null; } export type ViewerPresentationAction = @@ -101,6 +104,7 @@ export type ViewerPresentationAction = failure?: ViewerFailureCode | null; } | { type: "signal"; signal: SignalConnectionState } + | { type: "receive-failure"; revision: number; failure: "VIDEO_REJECTED" | null } | { type: "host"; host: ViewerHostState } | { type: "route"; @@ -159,6 +163,7 @@ export const INITIAL_VIEWER_PRESENTATION_STATE: ViewerPresentationState = { autoplayBlockedGeneration: null, playbackFailedGeneration: null, runtimeFailure: null, + receiveFailure: null, }; export function reduceViewerPresentation( @@ -181,6 +186,10 @@ export function reduceViewerPresentation( }; case "signal": return { ...state, signal: action.signal }; + case "receive-failure": + if (state.revision !== null && action.revision < state.revision) return state; + if (state.receiveFailure === action.failure) return state; + return { ...state, receiveFailure: action.failure }; case "host": { if (action.host === "offline" && state.host === "stopped") { return state; @@ -202,6 +211,7 @@ export function reduceViewerPresentation( return { ...state, revision: action.revision, + receiveFailure: action.authoritative ? null : state.receiveFailure, route: { revision: action.revision, phase: action.phase, @@ -303,6 +313,7 @@ export function reduceViewerPresentation( retainedFrame: false, routeStatus: null, playbackFailedGeneration: null, + receiveFailure: null, }; case "frame-proof-reset": if (!state.media || state.media.generation !== action.generation) { @@ -376,6 +387,7 @@ export function reduceViewerPresentation( autoplayBlockedGeneration: null, playbackFailedGeneration: null, runtimeFailure: null, + receiveFailure: null, }; case "server-error": return { ...state, runtimeFailure: "SERVER_ERROR" }; @@ -498,6 +510,9 @@ export function deriveViewerPresentation( // A failed route explains missing media only while the Host can publish. // Keep the fact: Host reconnection alone does not repair that route. if (state.host === "online" && state.routeStatus?.state === "failed") { + if (state.receiveFailure === "VIDEO_REJECTED") { + return presentation("video-rejected", "viewer.msg.videoRejected", frameOverlay, state, "VIDEO_REJECTED"); + } return presentation( "route-failed", "viewer.msg.routeFailed", diff --git a/src/client/media/viewer-sfu-route.ts b/src/client/media/viewer-sfu-route.ts index dc2b271d8..bcd26d222 100644 --- a/src/client/media/viewer-sfu-route.ts +++ b/src/client/media/viewer-sfu-route.ts @@ -85,6 +85,7 @@ interface ViewerSfuRouteEvents { state: "connected" | "reconnecting", revision: number, ) => void; + onSfuFailure?: (failure: "VIDEO_REJECTED" | null, revision: number) => void; send: (message: ClientMessage) => boolean; createSubscriber?: ( events: { @@ -93,7 +94,7 @@ interface ViewerSfuRouteEvents { onStats: (metrics: ConnectionMetrics) => void; onFirstDecodedFrame: () => boolean; onState: (state: "connected" | "reconnecting") => void; - onDisconnected: () => void; + onDisconnected: (failure?: "VIDEO_REJECTED") => void; }, ) => ViewerSubscriberTransport; } @@ -468,7 +469,7 @@ export class ViewerSfuRoute { this.events.onSfuState?.(state, slot.revision); } }, - onDisconnected: () => this.failSubscriberSlot(slot), + onDisconnected: (failure?: "VIDEO_REJECTED") => this.failSubscriberSlot(slot, failure), }; const subscriber = this.events.createSubscriber?.(subscriberEvents) ?? @@ -494,6 +495,7 @@ export class ViewerSfuRoute { activationToken: null, }; this.pending = slot; + this.events.onSfuFailure?.(null, slot.revision); try { const connected = await subscriber.connect(message); if (!connected) { @@ -906,7 +908,7 @@ export class ViewerSfuRoute { await disconnectSubscriber(active.subscriber); } - private handleFailure(slot: ViewerSubscriberSlot): void { + private handleFailure(slot: ViewerSubscriberSlot, failure?: "VIDEO_REJECTED"): void { if (this.closed || slot.failed) { return; } @@ -916,6 +918,7 @@ export class ViewerSfuRoute { slot.failed = true; return; } + this.events.onSfuFailure?.(failure ?? null, slot.revision); slot.failed = true; slot.subscriber.stopDecodedFrameProof(); if (this.pending === slot) { @@ -950,12 +953,12 @@ export class ViewerSfuRoute { } } - private failSubscriberSlot(slot: ViewerSubscriberSlot): void { + private failSubscriberSlot(slot: ViewerSubscriberSlot, failure?: "VIDEO_REJECTED"): void { if (this.pending !== slot && this.active !== slot) { slot.failed = true; return; } - this.handleFailure(slot); + this.handleFailure(slot, failure); } private routeFailed( diff --git a/src/client/native/media-bridge.ts b/src/client/native/media-bridge.ts index b04db9867..a50870b61 100644 --- a/src/client/native/media-bridge.ts +++ b/src/client/native/media-bridge.ts @@ -4,6 +4,7 @@ import { browserDebugEnabled, debugError, debugEvent } from "../lib/debug"; import { observeDebugConnection } from "../lib/debug-webrtc"; import type { ConnectionMetrics } from "../types"; import { preferScreenAudioStereo } from "../webrtc/screen-audio-sdp"; +import { requireReceivingVideo } from "../webrtc/video-codec"; import { collectConnectionMetrics, decodedVideoFrames, @@ -195,6 +196,7 @@ export class NativeMediaBridge { this.lifetime.signal.throwIfAborted(); const answer = preferScreenAudioStereo(await this.peer.createAnswer()); this.lifetime.signal.throwIfAborted(); + requireReceivingVideo(offer, answer); await this.peer.setLocalDescription(answer); await this.gatherCandidates(); this.lifetime.signal.throwIfAborted(); diff --git a/src/client/pages/ViewerPage.tsx b/src/client/pages/ViewerPage.tsx index 9207c3062..71db4c215 100644 --- a/src/client/pages/ViewerPage.tsx +++ b/src/client/pages/ViewerPage.tsx @@ -1093,6 +1093,11 @@ export function ViewerPage({ currentPeerMetrics: () => activePeerMetrics, qualityProbeEligible: () => qualityPresentationEligible(committedTransportConnected()), + onSfuFailure: (failure, revision) => { + if (active && viewerSfuRoute === route) { + dispatchPresentation({ type: "receive-failure", revision, failure }); + } + }, onSfuState: (state, revision) => { if (active && viewerSfuRoute === route) { const connected = state === "connected"; @@ -1288,6 +1293,7 @@ export function ViewerPage({ if (pendingPeer === probe) { if (snapshot.connectionId !== probe.candidateConnectionId) return; probe.snapshot = snapshot; + noteReceiveFailure(snapshot, probe.revision); if (probe.qualityProbe) { if ( pageSuspended || @@ -1319,6 +1325,7 @@ export function ViewerPage({ } provePendingPeer(); } else if (active && peerRef.current === peer) { + noteReceiveFailure(snapshot, currentRouteRevision); activePeerMetrics = snapshot.metrics; offerPeerQualityEvidence(snapshot, peer); setPeerSnapshot(snapshot); @@ -1407,6 +1414,7 @@ export function ViewerPage({ }, onUpdate: (snapshot) => { if (active) { + noteReceiveFailure(snapshot, currentRouteRevision); activePeerMetrics = snapshot.metrics; offerPeerQualityEvidence(snapshot, peer); if ( @@ -1454,6 +1462,11 @@ export function ViewerPage({ return peer; } + function noteReceiveFailure(snapshot: PeerSnapshot, revision: number): void { + dispatchPresentation({ type: "receive-failure", revision, + failure: snapshot.error?.key === "viewer.msg.videoRejected" ? "VIDEO_REJECTED" : null }); + } + async function handleMessage( message: ServerMessage, authorityToken: number, diff --git a/src/client/pages/status-preview-scenarios.ts b/src/client/pages/status-preview-scenarios.ts index 73501f3b0..0de019f91 100644 --- a/src/client/pages/status-preview-scenarios.ts +++ b/src/client/pages/status-preview-scenarios.ts @@ -104,6 +104,12 @@ export const STATUS_SCENARIOS: readonly StatusScenario[] = [ scenario("playback-failed", "浏览器播放失败", "连接存在,但播放器无法呈现画面。", [ ...receiving, { type: "playback-failed", generation: 1, revision: 1 }, ]), + scenario("video-rejected", "浏览器拒绝视频协商", "浏览器明确拒绝接收画面;沿用媒体恢复和呈现归属,不误报为网络线路故障。", [ + ...ready, + { type: "route", revision: 1, phase: "prepare", kind: "p2p" }, + { type: "receive-failure", revision: 1, failure: "VIDEO_REJECTED" }, + { type: "route-status", revision: 1, state: "failed" }, + ]), scenario("waiting-host", "等待房主开始分享", "房间存在,当前没有共享源。", [{ type: "access", access: "ready" }]), scenario("host-absence-snapshot", "重连后只确认房主不在线", "快照没有说明房主为何不在线;用中性等待,不推断主动停播或连接中断。", [ ...ready, { type: "host", host: "unknown" }, diff --git a/src/client/sfu/subscriber.ts b/src/client/sfu/subscriber.ts index a492ee250..85bf51b55 100644 --- a/src/client/sfu/subscriber.ts +++ b/src/client/sfu/subscriber.ts @@ -7,6 +7,7 @@ import { decodedVideoFrames, } from "../webrtc/stats"; import { preferScreenAudioStereo } from "../webrtc/screen-audio-sdp"; +import { requireReceivingVideo, VideoReceiveRejectedError } from "../webrtc/video-codec"; import { observeDecodedFrameProof } from "../media/decoded-frame-proof"; import { SfuPeer, type SfuConnectionConfig } from "./peer"; @@ -17,7 +18,7 @@ interface SubscriberEvents { onStats?: (metrics: ConnectionMetrics) => void; onFirstDecodedFrame?: () => boolean; onState?: (state: "connected" | "reconnecting") => void; - onDisconnected?: () => void; + onDisconnected?: (failure?: "VIDEO_REJECTED") => void; } interface SubscribedTrack { @@ -80,13 +81,13 @@ export class SfuSubscriber { throw new Error("SFU subscriber expected an offer"); await peer.waitForOperation(() => peer.pc.setRemoteDescription(description)); if (this.peer !== peer) return; - await peer.sendDescription( - preferScreenAudioStereo(await peer.waitForOperation(() => peer.pc.createAnswer())), - ); + const answer = preferScreenAudioStereo(await peer.waitForOperation(() => peer.pc.createAnswer())); + requireReceivingVideo(description, answer); + await peer.sendDescription(answer); if (peer.pc.connectionState === "connected") this.restartPending = false; }); - } catch { - if (this.peer === peer) this.fail(); + } catch (error) { + if (this.peer === peer) this.fail(error instanceof VideoReceiveRejectedError ? "VIDEO_REJECTED" : undefined); } } @@ -148,10 +149,10 @@ export class SfuSubscriber { this.streamEmitted = false; } - private fail(): void { + private fail(failure?: "VIDEO_REJECTED"): void { if (this.closed) return; void this.disconnect(); - this.events.onDisconnected?.(); + this.events.onDisconnected?.(failure); } private addTrack({ track, receiver }: RTCTrackEvent): void { diff --git a/src/client/ui/media-status.ts b/src/client/ui/media-status.ts index b31be001d..3bc3ac159 100644 --- a/src/client/ui/media-status.ts +++ b/src/client/ui/media-status.ts @@ -67,6 +67,7 @@ const STAGE_VISUALS = { playing: { tone: "live", icon: "play", comic: "media-playing" }, recovering: { tone: "warn", icon: "refresh", comic: "recovering", pulse: true }, "route-failed": { tone: "bad", icon: "wifiOff", comic: "route-failed" }, + "video-rejected": { tone: "bad", icon: "alert", comic: "playback-failed" }, "playback-failed": { tone: "bad", icon: "alert", comic: "playback-failed" }, "waiting-host": { tone: "off", icon: "moon", comic: "waiting-for-host" }, "host-offline": { tone: "bad", icon: "wifiOff", comic: "host-offline" }, diff --git a/src/client/webrtc/video-codec.ts b/src/client/webrtc/video-codec.ts index 2b416059c..22b7194af 100644 --- a/src/client/webrtc/video-codec.ts +++ b/src/client/webrtc/video-codec.ts @@ -1,3 +1,5 @@ +import { parse } from "sdp-transform"; + export type BrowserVideoCodec = "h264" | "vp8"; export type BrowserVideoCodecMode = "vp8" | "auto" | "h264"; @@ -135,3 +137,25 @@ export function normalizeVideoOfferSdp(sdp: string): string { }); return changed ? lines.join(sdp.includes("\r\n") ? "\r\n" : "\n") : sdp; } + +export class VideoReceiveRejectedError extends Error { + constructor() { super("Browser rejected the offered video"); } +} + +// JSEP rejects a media section with port zero. Let the Browser negotiate codecs; +// reject only an answer that explicitly declines every offered sending video. +export function requireReceivingVideo( + offer: RTCSessionDescriptionInit, + answer: RTCSessionDescriptionInit, +): void { + if (offer.type !== "offer" || answer.type !== "answer" || !offer.sdp || !answer.sdp) return; + const remote = parse(offer.sdp), local = parse(answer.sdp); + const sendingVideo = remote.media.flatMap((media, index) => { + const direction = media.direction ?? remote.direction; + return media.type === "video" && (media.port !== 0 || media.bundleOnly !== undefined) && + direction !== "inactive" && direction !== "recvonly" ? [index] : []; + }); + if (sendingVideo.length > 0 && sendingVideo.every(index => + local.media[index]?.type === "video" && local.media[index]?.port === 0, + )) throw new VideoReceiveRejectedError(); +} diff --git a/src/client/webrtc/viewer-peer.ts b/src/client/webrtc/viewer-peer.ts index cd3f70138..d5da408e0 100644 --- a/src/client/webrtc/viewer-peer.ts +++ b/src/client/webrtc/viewer-peer.ts @@ -14,6 +14,7 @@ import { type StatsAccumulator, } from "./stats"; import { preferScreenAudioStereo } from "./screen-audio-sdp"; +import { requireReceivingVideo, VideoReceiveRejectedError } from "./video-codec"; import { observeDecodedFrameProof } from "../media/decoded-frame-proof"; import { addRemoteIceCandidate, @@ -388,6 +389,8 @@ export class ViewerPeer implements ViewerMediaPeer { if (!this.isCurrentConnection(connection, connectionId)) { return; } + requireReceivingVideo(payload.description, answer); + if (this.snapshot) this.snapshot = { ...this.snapshot, error: null }; await this.waitForOperation(connection, () => connection.setLocalDescription(answer)); if (!this.isCurrentConnection(connection, connectionId)) { return; @@ -416,7 +419,8 @@ export class ViewerPeer implements ViewerMediaPeer { } private failOffer(error: unknown, parentPeerId: string, connectionId: string): void { - this.setError(error, "host.fail.connection"); + this.setError(error, error instanceof VideoReceiveRejectedError + ? "viewer.msg.videoRejected" : "host.fail.connection"); if ( this.recoveryOwner === "viewer" && !this.recoveryBudget.exhaustedReported && diff --git a/tests/native-media-bridge.test.ts b/tests/native-media-bridge.test.ts index 52efbc2b8..da5c6a281 100644 --- a/tests/native-media-bridge.test.ts +++ b/tests/native-media-bridge.test.ts @@ -124,6 +124,17 @@ afterEach(() => { }); describe("native media bridge", () => { + it("retires rejected local video through the existing bridge failure path", async () => { + const current = fixture(); + vi.mocked(current.control.prepareLocalEdge).mockResolvedValueOnce({ type: "offer", offerId: "local_offer_1", + sdp: "m=video 9 UDP/TLS/RTP/SAVPF 96\r\na=sendonly\r\n" }); + current.peer().createAnswer.mockResolvedValueOnce({ type: "answer", sdp: "m=video 0 UDP/TLS/RTP/SAVPF 120\r\n" }); + await expect(current.bridge.start()).rejects.toBeInstanceOf(NativeMediaBridgeError); + expect(current.control.acceptSignal).not.toHaveBeenCalled(); + expect(current.control.closeEdge).toHaveBeenCalledOnce(); + expect(current.peer().close).toHaveBeenCalledOnce(); + }); + it("recovers one retained transport after a disconnect grace, including a quiet source", async () => { vi.useFakeTimers(); const current = fixture(); diff --git a/tests/route-transition.test.ts b/tests/route-transition.test.ts index 516f9238d..fa26fce4b 100644 --- a/tests/route-transition.test.ts +++ b/tests/route-transition.test.ts @@ -1118,6 +1118,7 @@ describe("minimal route transition contracts", () => { }); it("ignores a retired subscriber failure after same-generation resync", async () => { + const onSfuFailure = vi.fn(); let releaseOldConnect!: () => void; const oldConnectGate = new Promise((resolve) => { releaseOldConnect = resolve; @@ -1125,6 +1126,7 @@ describe("minimal route transition contracts", () => { const messages: ClientMessage[] = []; const subscribers: ReturnType[] = []; const route = new ViewerSfuRoute("viewer_12345678", { + onSfuFailure, activatePeer: () => true, reconcileSfuChildren: () => undefined, onSfuStream: () => undefined, @@ -1182,6 +1184,12 @@ describe("minimal route transition contracts", () => { [], ); expect(subscribers[2]?.activate).toHaveBeenCalledOnce(); + onSfuFailure.mockClear(); + subscribers[1]?.events.onDisconnected("VIDEO_REJECTED"); + expect(onSfuFailure).not.toHaveBeenCalled(); + subscribers[2]?.events.onDisconnected("VIDEO_REJECTED"); + expect(onSfuFailure).toHaveBeenCalledExactlyOnceWith("VIDEO_REJECTED", 2); + await route.disconnect(); }); diff --git a/tests/sfu-client.test.ts b/tests/sfu-client.test.ts index fdce7d900..8d3fffb35 100644 --- a/tests/sfu-client.test.ts +++ b/tests/sfu-client.test.ts @@ -631,6 +631,20 @@ describe("embedded SFU browser transport", () => { expect(onStream).toHaveBeenLastCalledWith(null); }); + it("reports Browser video rejection through the existing SFU failure owner", async () => { + const send = vi.fn(() => true), onDisconnected = vi.fn(); + const subscriber = new SfuSubscriber({ send, onStream: vi.fn(), onDisconnected }); + cleanups.push(() => subscriber.disconnect()); + await subscriber.connect(config); + const pc = FakePc.instances[0]!; + pc.createAnswer.mockResolvedValueOnce({ type: "answer", sdp: "m=video 0 UDP/TLS/RTP/SAVPF 120\r\n" }); + await subscriber.acceptSignal(signal({ description: { type: "offer", + sdp: "m=video 9 UDP/TLS/RTP/SAVPF 96\r\na=sendonly\r\n" } })); + expect(onDisconnected).toHaveBeenCalledExactlyOnceWith("VIDEO_REJECTED"); + expect(pc.close).toHaveBeenCalledOnce(); + expect(send).not.toHaveBeenCalled(); + }); + it.each([ [false, false], [true, false], [false, true], [true, true], ])("retires an unfinished ICE restart on reauthentication (send=%s, answer=%s)", async (sent, answered) => { diff --git a/tests/video-codec.test.ts b/tests/video-codec.test.ts index 014d3bab7..a7eea66bc 100644 --- a/tests/video-codec.test.ts +++ b/tests/video-codec.test.ts @@ -1,5 +1,23 @@ import { expect, it } from "vitest"; -import { normalizeVideoOfferSdp } from "../src/client/webrtc/video-codec"; +import { normalizeVideoOfferSdp, requireReceivingVideo, VideoReceiveRejectedError } from "../src/client/webrtc/video-codec"; + +it("recognizes a Browser-rejected video answer without inferring codec support", () => { + const offer = (sdp: string): RTCSessionDescriptionInit => ({ type: "offer", sdp }); + const answer = (sdp: string): RTCSessionDescriptionInit => ({ type: "answer", sdp }); + const video = "m=video 9 UDP/TLS/RTP/SAVPF 96\r\na=sendonly\r\na=rtpmap:96 H264/90000\r\n"; + const rejected = "m=video 0 UDP/TLS/RTP/SAVPF 120\r\n"; + expect(() => requireReceivingVideo(offer(video), answer(rejected))).toThrow(VideoReceiveRejectedError); + for (const [offered, accepted] of [ + [video, video.replace("sendonly", "recvonly")], + [video + video, rejected + video.replace("sendonly", "recvonly")], + [video.replace("9 UDP", "0 UDP"), rejected], + [video.replace("sendonly", "inactive"), rejected], + [video.replace("sendonly", "recvonly"), rejected], + ["m=audio 9 UDP/TLS/RTP/SAVPF 111\r\n", "m=audio 0 UDP/TLS/RTP/SAVPF 111\r\n"], + ]) expect(() => requireReceivingVideo(offer(offered!), answer(accepted!))).not.toThrow(); + expect(() => requireReceivingVideo(offer(video.replace("9 UDP", "0 UDP") + "a=bundle-only\r\n"), + answer(rejected))).toThrow(VideoReceiveRejectedError); +}); it.each(["\r\n", "\n"])("removes orphan codec attributes within each offered video section (%j)", newline => { const lines = [ diff --git a/tests/viewer-peer-generation.test.ts b/tests/viewer-peer-generation.test.ts index fcef6b920..db3d440f9 100644 --- a/tests/viewer-peer-generation.test.ts +++ b/tests/viewer-peer-generation.test.ts @@ -35,6 +35,7 @@ const ANSWER_SDP = [ interface ConnectionPlan { constructorError?: Error; answerError?: Error; + answerSdp?: string; candidateGates?: Promise[]; localDescriptionGate?: Promise; statsGate?: Promise; @@ -75,6 +76,7 @@ class FakePeerConnection extends EventTarget { private readonly candidateGates: Promise[]; private readonly localDescriptionGate: Promise | null; private readonly answerError: Error | null; + private readonly answerSdp: string; readonly addIceCandidate = vi.fn(async (_candidate: RTCIceCandidateInit | null) => { const gate = this.candidateGates.shift(); @@ -107,6 +109,7 @@ class FakePeerConnection extends EventTarget { this.candidateGates = [...(plan.candidateGates ?? [])]; this.localDescriptionGate = plan.localDescriptionGate ?? null; this.answerError = plan.answerError ?? null; + this.answerSdp = plan.answerSdp ?? ANSWER_SDP; this.statsGate = plan.statsGate ?? null; FakePeerConnection.instances.push(this); } @@ -123,7 +126,7 @@ class FakePeerConnection extends EventTarget { } return { type: "answer", - sdp: ANSWER_SDP, + sdp: this.answerSdp, }; } @@ -344,6 +347,23 @@ describe("ViewerPeer connection generations", () => { peer.dispose(); }); + it("returns rejected video to the existing route owner with its receiving cause", async () => { + const restart = vi.fn(() => true), exhausted = vi.fn(() => true), sendSignal = vi.fn(() => true); + const updates: PeerSnapshot[] = []; + const peer = new ViewerPeer({ iceServers: [] }, { sendSignal, sendRestartRequest: restart, + onStream: () => undefined, onUpdate: value => updates.push(value), onRecoveryExhausted: exhausted }, + { recoveryOwner: "route" }); + FakePeerConnection.plans.push({ answerSdp: "m=video 0 UDP/TLS/RTP/SAVPF 120\r\n" }); + await peer.acceptSignal("host", { kind: "description", connectionId: "rejected", + description: { type: "offer", sdp: "m=video 9 UDP/TLS/RTP/SAVPF 96\r\na=sendonly\r\n" } }); + expect(updates.at(-1)?.error).toEqual({ key: "viewer.msg.videoRejected" }); + expect(exhausted).toHaveBeenCalledExactlyOnceWith("host", "rejected"); + expect(sendSignal).not.toHaveBeenCalled(); + expect(restart).not.toHaveBeenCalled(); + expect(timeoutCallbacks.size).toBe(0); + peer.dispose(); + }); + it("answers after a rejected queued candidate and still applies later candidates", async () => { const signals: SignalPayload[] = []; const snapshots: PeerSnapshot[] = []; diff --git a/tests/viewer-presentation.test.ts b/tests/viewer-presentation.test.ts index f3c117854..04657c668 100644 --- a/tests/viewer-presentation.test.ts +++ b/tests/viewer-presentation.test.ts @@ -23,6 +23,37 @@ function applyFrom( } describe("Viewer presentation reducer", () => { + it("retains receiving rejection through route exhaustion but not fresh negotiation or stop", () => { + const failed = apply( + { type: "access", access: "ready" }, { type: "host", host: "online" }, + { type: "route", revision: 2, phase: "prepare", kind: "p2p" }, + { type: "receive-failure", revision: 2, failure: "VIDEO_REJECTED" }, + { type: "route", revision: 3, phase: "active", kind: "none" }, + { type: "route-status", revision: 3, state: "failed" }, + ); + expect(deriveViewerPresentation(failed)).toMatchObject({ stage: "video-rejected", failureCode: "VIDEO_REJECTED" }); + const late = applyFrom(failed, { type: "receive-failure", revision: 2, failure: null }); + expect(late).toBe(failed); + expect(deriveViewerPresentation(applyFrom(failed, { type: "host", host: "offline" })).stage).toBe("host-offline"); + expect(deriveViewerPresentation(applyFrom(failed, + { type: "receive-failure", revision: 3, failure: null })).stage).toBe("route-failed"); + expect(applyFrom(failed, { type: "sharing-stopped" }).receiveFailure).toBeNull(); + expect(applyFrom(failed, { type: "route", revision: 1, phase: "active", kind: "p2p", authoritative: true }) + .receiveFailure).toBeNull(); + }); + + it("does not replace a healthy picture with a rejected preparation", () => { + const state = apply( + { type: "access", access: "ready" }, { type: "host", host: "online" }, + { type: "media-bound", generation: 1, revision: 1 }, + { type: "frame-presented", generation: 1, proofEpoch: 0, revision: 1 }, + { type: "route", revision: 2, phase: "prepare", kind: "sfu" }, + { type: "receive-failure", revision: 2, failure: "VIDEO_REJECTED" }, + ); + expect(deriveViewerPresentation(state).stage).toBe("playing"); + expect(applyFrom(state, { type: "frame-presented", generation: 1, proofEpoch: 0, revision: 2 }).receiveFailure).toBeNull(); + }); + it("derives connection stages from typed current-revision facts", () => { const preparing = apply( { type: "access", access: "ready" }, From 2f9e7712ea48bcfa1b0d1dbcb21f2223800b38cc Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 07:08:56 +0800 Subject: [PATCH 102/262] docs(network): record native admission startup boundary --- docs/research/nat-traversal.md | 10 ++++++++++ docs/todo.md | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 7ddcdb0f1..50e0de4b5 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -888,6 +888,16 @@ the roughly one-second grant acknowledgment, reaches eight and receives no RTP within the unchanged bound. Complete-SDP and early-trickle controls both fail. The early-trickle counterfactual is opt-in and intentionally fails its media check. +The simpler pre-ICE control uses supported `DialUDPPort` in both Native +listener/dialer orientations. With keys already admitted, the real Native control +session receives fresh RTP, replaces a flow without interrupting its sibling and +retires all flows on stop or control close. All four Linux race cases and vet +pass with `PrepareUDPPort` removed from the copied dependency, without an explicit +diagnostic ping or an admission-dependent candidate gate. The other isolated +dependency corrections remain. This removes the need for that extra startup API +when key admission precedes media negotiation; it does not implement room-scoped +key exchange or demonstrate higher direct reachability. + This establishes a startup-order constraint, not a product integration or a Piik field defect. Keys and grants remain local fixture inputs; the candidate gate and unsupported dependency method are not accepted product mechanisms. diff --git a/docs/todo.md b/docs/todo.md index ad7ee5107..2456ac601 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -88,8 +88,8 @@ Implementation needs a concrete useful result within the product model. from exhaustion by sender behavior. No new product setting or flow owner is accepted. Resolve key exchange and admission before ICE at the assigned-connection owner; - research owns the Native control retirement checks and the isolated candidate - gate. Neither the pre-granted nor gated fixture implements room authorization. + research owns the supported pre-ICE dial/retirement checks and the isolated + gated counterfactual. Neither fixture implements room authorization. Keep isolated dependency patches and recovery counterfactuals outside product routing; a diagnostic reply is not media-readiness authority. Research owns the experiments and their limits. From dabc7a5afe049a5b28d2dfd3bba53f2cce73ada6 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 07:22:59 +0800 Subject: [PATCH 103/262] docs(network): record room-scoped key signaling evidence --- docs/research/nat-traversal.md | 13 ++++++++++++- docs/todo.md | 4 +++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 50e0de4b5..428025830 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -898,8 +898,19 @@ dependency corrections remain. This removes the need for that extra startup API when key admission precedes media negotiation; it does not implement room-scoped key exchange or demonstrate higher direct reachability. +An isolated Go signaling extension now carries one bounded public key through +the real room WebSocket and existing assigned-connection authorizer. It forwards +candidate and retained-connection metadata, rejects unrelated peers/rooms and +retired connection IDs, and cannot adopt a new connection from a key message. +Ordinary invitation revocation still closes the authenticated Viewer session. +The protocol/signaling race suites pass; removing the connection guard fails +the scoped probe. This establishes metadata authorization only: Native key +installation and flow retirement are not yet connected to those room events. +The generated extension adds no server key registry and does not change the +product wire format or Browser implementation. + This establishes a startup-order constraint, not a product integration or a Piik -field defect. Keys and grants remain local fixture inputs; the candidate gate +field defect. The Native media checks still use local grants; the candidate gate and unsupported dependency method are not accepted product mechanisms. The existing assigned-connection owner must resolve key exchange and admission before ICE starts. Accepting arbitrary keys, passing room credentials into Native diff --git a/docs/todo.md b/docs/todo.md index 2456ac601..6d0e3b1d4 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -89,7 +89,9 @@ Implementation needs a concrete useful result within the product model. accepted. Resolve key exchange and admission before ICE at the assigned-connection owner; research owns the supported pre-ICE dial/retirement checks and the isolated - gated counterfactual. Neither fixture implements room authorization. + gated counterfactual. A Go-only overlay verifies room-scoped key signaling; + Native key installation and flow revocation still need to follow those room + events. No product wire extension or Browser integration is implemented. Keep isolated dependency patches and recovery counterfactuals outside product routing; a diagnostic reply is not media-readiness authority. Research owns the experiments and their limits. From 49edc9c6647af948c82f0a7e13b5126b593dc013 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 07:38:54 +0800 Subject: [PATCH 104/262] docs(network): record native flow and relay policy boundaries --- docs/research/nat-traversal.md | 20 ++++++++++++++++++++ docs/todo.md | 5 ++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 428025830..93a3454b7 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -366,6 +366,15 @@ UDP pairs within the observation window. Tailcat delivered those cases while the fixture relay remained available. This demonstrates relay coverage, not an additional P2P reachability class or a field success rate. +Relay policy remains a separate integration boundary. Tailcat's documented +fallback may carry media through DERP during startup or recovery. Closing DERP +or counting its packets in a fixture measures the path; neither enforces Piik's +product policy. The reviewed Tailcat API does not expose a direct-only media +policy that retains discovery. Disabling DERP also removes bootstrap/endpoint +exchange, and polling a direct-path status cannot authorize each later packet. +Product adoption needs that separation or an accepted transport-policy change; +it must not silently add a media relay to P2P-only rooms or bypass SFU-only sites. + Replacing one gateway mapping without notifying endpoints separated recovery from initial reachability. Neither established direct path delivered within 20 seconds with the relay closed. Fresh Pion discovery/exchange recovered in @@ -909,6 +918,17 @@ installation and flow retirement are not yet connected to those room events. The generated extension adds no server key registry and does not change the product wire format or Browser implementation. +A separate Native control check derives admission from its actual owned inbound +flows. Closing one flow retains the shared peer's authorization while another +flow remains; closing the last disconnects that peer. Final revocation and a new +grant serialize without holding the flow-map lock through Tailcat callbacks. +Stop and control-close cases pass under the race detector, including fresh +sibling RTP before preparing a replacement. Counterfactuals that disconnect on +every close or never disconnect fail those checks. This composes physical-flow +cleanup with the Native session; keys still come from the local fixture, not +the room signaling experiment above. Complete room-to-App authorization remains +open. + This establishes a startup-order constraint, not a product integration or a Piik field defect. The Native media checks still use local grants; the candidate gate and unsupported dependency method are not accepted product mechanisms. diff --git a/docs/todo.md b/docs/todo.md index 6d0e3b1d4..6f38b2d79 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -90,8 +90,11 @@ Implementation needs a concrete useful result within the product model. Resolve key exchange and admission before ICE at the assigned-connection owner; research owns the supported pre-ICE dial/retirement checks and the isolated gated counterfactual. A Go-only overlay verifies room-scoped key signaling; - Native key installation and flow revocation still need to follow those room + a separate Native control fixture ties key revocation to its last owned flow. + Native key installation and flow retirement still need to follow those room events. No product wire extension or Browser integration is implemented. + Resolve direct-only media enforcement while retaining discovery; DERP must + not silently become another media fallback or bypass site/share route policy. Keep isolated dependency patches and recovery counterfactuals outside product routing; a diagnostic reply is not media-readiness authority. Research owns the experiments and their limits. From 0225dbacbad2bb0c64978d9dc582349b96ce2ba7 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 08:18:31 +0800 Subject: [PATCH 105/262] docs(network): retain direct media startup boundary Record the isolated relay-policy controls and unresolved Pion and Native cold-start timing without accepting a product transport or retry change. --- docs/research/nat-traversal.md | 34 ++++++++++++++++++++++++++++++++-- docs/todo.md | 8 ++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 93a3454b7..b30b5a32a 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -372,8 +372,38 @@ or counting its packets in a fixture measures the path; neither enforces Piik's product policy. The reviewed Tailcat API does not expose a direct-only media policy that retains discovery. Disabling DERP also removes bootstrap/endpoint exchange, and polling a direct-path status cannot authorize each later packet. -Product adoption needs that separation or an accepted transport-policy change; -it must not silently add a media relay to P2P-only rooms or bypass SFU-only sites. + +An isolated packet-policy overlay separates discovery and fixed +[WireGuard handshake messages](https://www.wireguard.com/protocol/#key-exchange-and-data-packets) +from encapsulated data. Blocked UDP carries no media; a relay-allowed peer's +incoming data is also rejected. Removing either direction's guard fails its +corresponding control. The ordinary-NAT Pion fixture delivered fresh direct SRTP +in an initial run but failed cold startup on a repeat, so it is not a passing +acceptance gate. +The relay counts encrypted data envelopes without recording keys or payloads; +ordinary relay delivery validates that counter. The stricter variant that also +blocks WireGuard handshakes leaves inner ICE connecting without media after +20 seconds, even after the outer endpoint is known. Native control replacement, +sibling continuity and final retirement pass with the media-only restriction +on loopback. These checks do not establish Native startup through NAT. + +The complete Native Engine/receiver/playback fixture also fails cold startup through +ordinary kernel NAT with this restriction, before any address move. Its +relay-allowed control passes and retains fresh playback after DERP closes. +Discovery traces show one direction's initial ping unanswered: one endpoint +confirms direct access promptly, while the other confirms it around six seconds +later. Native's eight initial ICE writes end around 1.4 seconds; a pending UDP +flow may still be unaccepted then. The late direct path does not revive that +exhausted generation. The late side can be either endpoint. A one-shot supported +`Client.DiscoPing` before ICE returns the relay result in milliseconds and does +not fix startup. Diagnostic replies and successful key admission therefore +cannot stand in for bidirectional transport preparation. + +Authentication, key handling, retry timers and Pion budgets remain unchanged. +The experiment narrows the missing direct-media preparation contract; it is not +a supported upstream option, passing adoption gate or improvement to +direct NAT reachability. Product adoption must preserve site/share policy, +including P2P-only and SFU-only rooms, without silently adding another relay. Replacing one gateway mapping without notifying endpoints separated recovery from initial reachability. Neither established direct path delivered within diff --git a/docs/todo.md b/docs/todo.md index 6f38b2d79..877e25639 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -93,8 +93,12 @@ Implementation needs a concrete useful result within the product model. a separate Native control fixture ties key revocation to its last owned flow. Native key installation and flow retirement still need to follow those room events. No product wire extension or Browser integration is implemented. - Resolve direct-only media enforcement while retaining discovery; DERP must - not silently become another media fallback or bypass site/share route policy. + Obtain a supported direct-media policy that retains discovery/handshakes; + the isolated packet-policy control is not a product API. Resolve its cold-start + failures in Pion and Native fixtures: direct discovery can complete after + initial ICE checks are exhausted. One diagnostic ping before ICE does not fix it. + DERP must not silently become another media fallback or bypass site/share + route policy. Keep isolated dependency patches and recovery counterfactuals outside product routing; a diagnostic reply is not media-readiness authority. Research owns the experiments and their limits. From 4a10af77ecfa2f9c4057a71ecc636b26fd202586 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 08:33:15 +0800 Subject: [PATCH 106/262] docs(network): record preparation cost and remaining integration gates --- docs/research/nat-traversal.md | 21 ++++++++++++++++++--- docs/todo.md | 6 ++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index b30b5a32a..ea0db5ece 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -387,9 +387,10 @@ blocks WireGuard handshakes leaves inner ICE connecting without media after sibling continuity and final retirement pass with the media-only restriction on loopback. These checks do not establish Native startup through NAT. -The complete Native Engine/receiver/playback fixture also fails cold startup through -ordinary kernel NAT with this restriction, before any address move. Its -relay-allowed control passes and retains fresh playback after DERP closes. +Without another preparation step, the complete Native Engine/receiver/playback +fixture also fails cold startup through ordinary kernel NAT with this restriction, +before any address move. Its relay-allowed control passes and retains fresh +playback after DERP closes. Discovery traces show one direction's initial ping unanswered: one endpoint confirms direct access promptly, while the other confirms it around six seconds later. Native's eight initial ICE writes end around 1.4 seconds; a pending UDP @@ -399,6 +400,20 @@ exhausted generation. The late side can be either endpoint. A one-shot supported not fix startup. Diagnostic replies and successful key admission therefore cannot stand in for bidirectional transport preparation. +A separate startup control uses supported Tailcat TCP establishment inside the +same tunnel before Native ICE, within the existing eight-second preparation +deadline. The server checks the assigned peer key and both sides close the +temporary TCP flow; media still uses UDP. With relayed data prohibited, ordinary +NAT reaches actual Native receiver/local playback, while blocked UDP expires +before Native emits SDP or ICE. Relay data counts remain zero throughout. +Preparation costs about seven seconds in the repeated local runs. This exceeds +the spare startup time in the existing handoff check: its fifteen seconds of +post-start observation do not finish inside the unchanged twenty-second bound, +although fresh playback continues after DERP closes. The startup check therefore +does not establish migration acceptance or justify extending product deadlines. +The extra listener and delay are integration costs, not an accepted product +readiness mechanism or evidence of improved NAT reachability. + Authentication, key handling, retry timers and Pion budgets remain unchanged. The experiment narrows the missing direct-media preparation contract; it is not a supported upstream option, passing adoption gate or improvement to diff --git a/docs/todo.md b/docs/todo.md index 877e25639..105e695d0 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -95,8 +95,10 @@ Implementation needs a concrete useful result within the product model. events. No product wire extension or Browser integration is implemented. Obtain a supported direct-media policy that retains discovery/handshakes; the isolated packet-policy control is not a product API. Resolve its cold-start - failures in Pion and Native fixtures: direct discovery can complete after - initial ICE checks are exhausted. One diagnostic ping before ICE does not fix it. + preparation cost and full handoff boundary: direct discovery can finish after + initial ICE checks are exhausted, and the working isolated TCP preparation + consumes most of the existing preparation budget. Research owns the controls; + neither diagnostic replies nor startup alone establish recovery acceptance. DERP must not silently become another media fallback or bypass site/share route policy. Keep isolated dependency patches and From 9fad61712c79d9f7af9a4e0c4e2ce966b1525db5 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 08:56:04 +0800 Subject: [PATCH 107/262] docs(network): record bounded discovery repair and integration limits --- docs/research/nat-traversal.md | 36 ++++++++++++++++++++++------------ docs/todo.md | 10 ++++------ 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index ea0db5ece..3dc8c79dc 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -400,19 +400,29 @@ exhausted generation. The late side can be either endpoint. A one-shot supported not fix startup. Diagnostic replies and successful key admission therefore cannot stand in for bidirectional transport preparation. -A separate startup control uses supported Tailcat TCP establishment inside the -same tunnel before Native ICE, within the existing eight-second preparation -deadline. The server checks the assigned peer key and both sides close the -temporary TCP flow; media still uses UDP. With relayed data prohibited, ordinary -NAT reaches actual Native receiver/local playback, while blocked UDP expires -before Native emits SDP or ICE. Relay data counts remain zero throughout. -Preparation costs about seven seconds in the repeated local runs. This exceeds -the spare startup time in the existing handoff check: its fifteen seconds of -post-start observation do not finish inside the unchanged twenty-second bound, -although fresh playback continues after DERP closes. The startup check therefore -does not establish migration acceptance or justify extending product deadlines. -The extra listener and delay are integration costs, not an accepted product -readiness mechanism or evidence of improved NAT reachability. +The producer/consumer review identifies a bootstrap contract mismatch: +[Tailcat's initial endpoint advertisement](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/tailcat.go#L1570) +uses CallMeMaybe without first sending direct UDP, while +[magicsock's receiver](https://github.com/tailscale/tailscale/blob/a0e471a35b8f/wgengine/magicsock/endpoint.go#L2146) +assumes that send has already opened the peer's stateful firewall. A dependency +candidate follows pings to newly learned endpoints with the existing endpoint +advertisement. Identical advertisements do not cause another reply, and no +timer, retry count or cryptographic check changes. + +With that candidate, three fresh Pion starts deliver direct SRTP within +68–87 ms of their observation start; blocked-UDP and incoming-relay controls +still enforce the media policy. The real Windows VP8 encoder, Native receiver +and actual Chromium/Firefox bridge also retain fresh decoded pictures through +both local-address handoff directions and subsequent DERP closure, within the +unchanged observation bound. The Native fixture counts relayed data from before +bootstrap, not only after direct-path discovery. These are isolated, generated- +media checks; full App/room integration and physical networks remain open. + +A separate TCP preparation control restored startup at about seven seconds of +extra cost but exceeded the spare time in the full handoff check. That extra +flow is not retained; the library bootstrap candidate supersedes it. Fixed-frame +or bare-Pion playback controls still expose the separate encoder and local-bridge +boundaries described below. Authentication, key handling, retry timers and Pion budgets remain unchanged. The experiment narrows the missing direct-media preparation contract; it is not diff --git a/docs/todo.md b/docs/todo.md index 105e695d0..977c07157 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -93,12 +93,10 @@ Implementation needs a concrete useful result within the product model. a separate Native control fixture ties key revocation to its last owned flow. Native key installation and flow retirement still need to follow those room events. No product wire extension or Browser integration is implemented. - Obtain a supported direct-media policy that retains discovery/handshakes; - the isolated packet-policy control is not a product API. Resolve its cold-start - preparation cost and full handoff boundary: direct discovery can finish after - initial ICE checks are exhausted, and the working isolated TCP preparation - consumes most of the existing preparation budget. Research owns the controls; - neither diagnostic replies nor startup alone establish recovery acceptance. + Obtain supported direct-media and initial endpoint-discovery contracts; + the isolated packet-policy overlay and bootstrap repair are not product APIs. + Carry the repaired library ordering through the complete room/App composition; + neither diagnostic replies nor isolated handoffs establish full recovery. DERP must not silently become another media fallback or bypass site/share route policy. Keep isolated dependency patches and From 0f669d697fa24b29f38fa7f9899ad6baf942c0d4 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:09:57 +0800 Subject: [PATCH 108/262] docs(network): record composed room and native admission evidence --- docs/research/nat-traversal.md | 30 +++++++++++++++++++----------- docs/todo.md | 9 +++++---- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 3dc8c79dc..369229705 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -968,10 +968,9 @@ candidate and retained-connection metadata, rejects unrelated peers/rooms and retired connection IDs, and cannot adopt a new connection from a key message. Ordinary invitation revocation still closes the authenticated Viewer session. The protocol/signaling race suites pass; removing the connection guard fails -the scoped probe. This establishes metadata authorization only: Native key -installation and flow retirement are not yet connected to those room events. -The generated extension adds no server key registry and does not change the -product wire format or Browser implementation. +the scoped probe. This establishes metadata authorization. The generated +extension adds no server key registry and does not change the product wire +format or Browser implementation. A separate Native control check derives admission from its actual owned inbound flows. Closing one flow retains the shared peer's authorization while another @@ -980,13 +979,22 @@ grant serialize without holding the flow-map lock through Tailcat callbacks. Stop and control-close cases pass under the race detector, including fresh sibling RTP before preparing a replacement. Counterfactuals that disconnect on every close or never disconnect fail those checks. This composes physical-flow -cleanup with the Native session; keys still come from the local fixture, not -the room signaling experiment above. Complete room-to-App authorization remains -open. - -This establishes a startup-order constraint, not a product integration or a Piik -field defect. The Native media checks still use local grants; the candidate gate -and unsupported dependency method are not accepted product mechanisms. +cleanup with the Native session; keys in that control remain local fixture inputs. + +The composed room/Native control passes actual room-authorized key exchange, +SDP/candidates and RTP delivery, same-parent replacement after DERP closure, +late old-receiver cleanup and invitation-driven final retirement. Rebuilding +uses the current assignment to exchange the key before ICE; the subsequent SDP +offer adopts the replacement ID. Trying to exchange a key on the new ID first +is rejected. Thus pre-ICE preparation does not require giving metadata the power +to adopt a connection. The fixture still supplies the tunnel bootstrap address +and translates the room event into the Native stop command; complete Browser/App +authorization and transport selection remain open. This check establishes +packet reception and resource retirement, not decoded playback or field benefit. + +These checks establish startup-order and ownership constraints, not a product +integration or a Piik field defect. The earlier candidate gate and unsupported +dependency method are not accepted product mechanisms. The existing assigned-connection owner must resolve key exchange and admission before ICE starts. Accepting arbitrary keys, passing room credentials into Native control or enlarging its timeout is not an equivalent solution. diff --git a/docs/todo.md b/docs/todo.md index 977c07157..3f289209c 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -89,10 +89,11 @@ Implementation needs a concrete useful result within the product model. accepted. Resolve key exchange and admission before ICE at the assigned-connection owner; research owns the supported pre-ICE dial/retirement checks and the isolated - gated counterfactual. A Go-only overlay verifies room-scoped key signaling; - a separate Native control fixture ties key revocation to its last owned flow. - Native key installation and flow retirement still need to follow those room - events. No product wire extension or Browser integration is implemented. + gated counterfactual. The Go-only room/Native composition verifies authorized + key exchange, same-parent replacement and invitation-driven retirement; + bootstrap addresses and event-to-control dispatch remain fixture inputs. + Integrate those boundaries at the existing Browser/App connection owners. + No product wire extension or Browser integration is implemented. Obtain supported direct-media and initial endpoint-discovery contracts; the isolated packet-policy overlay and bootstrap repair are not product APIs. Carry the repaired library ordering through the complete room/App composition; From 2c56580312c8422a1187d052616823259d8b7c02 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:15:32 +0800 Subject: [PATCH 109/262] docs(network): record pending-flow isolation evidence --- docs/research/nat-traversal.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 369229705..253f78f76 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -980,6 +980,9 @@ Stop and control-close cases pass under the race detector, including fresh sibling RTP before preparing a replacement. Counterfactuals that disconnect on every close or never disconnect fail those checks. This composes physical-flow cleanup with the Native session; keys in that control remain local fixture inputs. +The pending-flow adapter also rejects a different admitted peer without consuming +the assigned listener. The two-peer control reproduced the former premature +close; the repair permits the expected peer or cancels through the same owner. The composed room/Native control passes actual room-authorized key exchange, SDP/candidates and RTP delivery, same-parent replacement after DERP closure, From cf06f29c1ab6b15a974d7b35b758aadb4ca3c85a Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:31:53 +0800 Subject: [PATCH 110/262] docs(network): record scoped bootstrap exchange boundary --- docs/research/nat-traversal.md | 20 ++++++++++++++------ docs/todo.md | 3 ++- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 253f78f76..d6a2573e3 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -962,7 +962,7 @@ dependency corrections remain. This removes the need for that extra startup API when key admission precedes media negotiation; it does not implement room-scoped key exchange or demonstrate higher direct reachability. -An isolated Go signaling extension now carries one bounded public key through +An isolated Go signaling extension now carries bounded bootstrap metadata through the real room WebSocket and existing assigned-connection authorizer. It forwards candidate and retained-connection metadata, rejects unrelated peers/rooms and retired connection IDs, and cannot adopt a new connection from a key message. @@ -984,16 +984,24 @@ The pending-flow adapter also rejects a different admitted peer without consumin the assigned listener. The two-peer control reproduced the former premature close; the repair permits the expected peer or cancels through the same owner. -The composed room/Native control passes actual room-authorized key exchange, +The composed room/Native control passes actual room-authorized bootstrap exchange, SDP/candidates and RTP delivery, same-parent replacement after DERP closure, late old-receiver cleanup and invitation-driven final retirement. Rebuilding uses the current assignment to exchange the key before ICE; the subsequent SDP offer adopts the replacement ID. Trying to exchange a key on the new ID first is rejected. Thus pre-ICE preparation does not require giving metadata the power -to adopt a connection. The fixture still supplies the tunnel bootstrap address -and translates the room event into the Native stop command; complete Browser/App -authorization and transport selection remain open. This check establishes -packet reception and resource retirement, not decoded playback or field benefit. +to adopt a connection. Listener metadata exchanges the node and discovery keys, +a nonzero shared key and a bounded port; it carries no relay URLs or region IDs. +The dialer constructs the upstream address from received fields and a trusted +local relay configuration. Strict shape checks and the existing route diagnostic +owner keep connection material out of its output; complete protocol/signaling +race suites and the direct-media composition pass. +Trusted relay configuration and room-event dispatch to Native control remain +fixture inputs. Complete Browser/App authorization and transport selection remain +open: the current Native-capable Viewer chooses its backend on the SDP offer, +so it does not yet advertise Native transport availability before negotiation. +This check establishes packet reception and resource retirement, not decoded +playback or field benefit. These checks establish startup-order and ownership constraints, not a product integration or a Piik field defect. The earlier candidate gate and unsupported diff --git a/docs/todo.md b/docs/todo.md index 3f289209c..e4ceeee53 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -91,7 +91,8 @@ Implementation needs a concrete useful result within the product model. research owns the supported pre-ICE dial/retirement checks and the isolated gated counterfactual. The Go-only room/Native composition verifies authorized key exchange, same-parent replacement and invitation-driven retirement; - bootstrap addresses and event-to-control dispatch remain fixture inputs. + connection material now travels through that assigned-connection owner; + trusted relay configuration and event-to-control dispatch remain fixture inputs. Integrate those boundaries at the existing Browser/App connection owners. No product wire extension or Browser integration is implemented. Obtain supported direct-media and initial endpoint-discovery contracts; From 1a6816316949b905e11a94e842c3c7a3efa15227 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 10:01:06 +0800 Subject: [PATCH 111/262] docs(network): record session capability experiment boundary --- docs/research/nat-traversal.md | 19 +++++++++++++++++++ docs/todo.md | 5 ++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index d6a2573e3..828f94ac9 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1003,6 +1003,25 @@ so it does not yet advertise Native transport availability before negotiation. This check establishes packet reception and resource retirement, not decoded playback or field benefit. +An isolated session-capability candidate addresses that timing boundary without +a second exchange or room-wide key registry. Viewer authentication supplies its +public Native receiver key; the existing preparing and committed assignments +project it only to assigned parents. Public presence and unrelated participants +receive none. The existing restart request can replace or clear the key only +for the current physical connection; a retired request cannot change it. +Invitation revocation removes it with the authenticated session. Browser-only +and Native cases pass through real room WebSockets under the race detector. +The actual App key producer and sender/receiver consumers are still unimplemented; +advertised availability is a fixture input, not a new media-readiness authority. + +`room-browser.mjs` bundles the real Browser schemas and diagnostics with matching +experimental fields. It accepts the Go fixture's twelve route/restart messages, +rejects malformed capability/listener fields and removes scoped keys from both +console events and exported reports while retaining connection and keyframe +evidence. The Go and Browser optional-map null checks agree. This is a Node-run +producer/consumer check, not Browser UI or complete App transport acceptance. +The product wire and diagnostic implementation remain unchanged. + These checks establish startup-order and ownership constraints, not a product integration or a Piik field defect. The earlier candidate gate and unsupported dependency method are not accepted product mechanisms. diff --git a/docs/todo.md b/docs/todo.md index e4ceeee53..8c79104b1 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -93,7 +93,10 @@ Implementation needs a concrete useful result within the product model. key exchange, same-parent replacement and invitation-driven retirement; connection material now travels through that assigned-connection owner; trusted relay configuration and event-to-control dispatch remain fixture inputs. - Integrate those boundaries at the existing Browser/App connection owners. + The isolated authenticated-session capability projection and Browser schema/ + diagnostic checks pass; App key production, early discovery and transport + consumption remain open. Integrate those boundaries at the existing + Browser/App connection owners. No product wire extension or Browser integration is implemented. Obtain supported direct-media and initial endpoint-discovery contracts; the isolated packet-policy overlay and bootstrap repair are not product APIs. From cacad624155f82e9e282a22dd9c80d1455e99104 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 10:37:33 +0800 Subject: [PATCH 112/262] docs(network): record native socket ownership boundary --- docs/research/nat-traversal.md | 23 ++++++++++++++++++++--- docs/todo.md | 4 +++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 828f94ac9..5b76b5b8c 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -742,9 +742,16 @@ remain prerequisites for product adoption. The remote-socket overlay extends that comparison through the actual Native encoded source, sender, receiver and local playback. It leaves product files -untouched and admits one fixture peer flow per engine. Its fixed 8×8 input -exposes forwarding pauses but cannot establish live encoder recovery; declaring -those fixed bytes rate-controlled is only an ablation. +untouched and binds the supplied socket to each connection-creation request. +Its fixed 8×8 input exposes forwarding pauses but cannot establish live encoder +recovery; declaring those fixed bytes rate-controlled is only an ablation. +The fixed-frame Viewer-address handoff pauses with both the former and current +socket adapters, including a matched control with isolated loopback playback. +Under that same playback isolation, the real Windows VP8/FFmpeg control passes +with the current adapter in all three checked cases: ordinary unchanged, source +address change and Viewer address change. These controls separate the adapter +change from the fixed-input recovery failure; they do not establish its complete +cause or justify relaxing media acceptance. The live control instead uses the Windows VP8 helper, Native output-plan owner and final FFmpeg decode. The top quarter of the input changes from red to blue @@ -846,6 +853,16 @@ both remaining flows. Omitting receiver retirement in the test overlay is detected. Keys are still pre-granted locally: this establishes resource cleanup, not room authorization, automatic App recovery or Browser decoding in this check. +The request-scoped adapter removes the former Engine-wide next-socket slot and +temporary bootstrap receiver. A lazy factory reaches the existing physical +socket owner only after input/capacity admission; reuse does not allocate again. +Unsupported-codec failure, independent receiver retirement, control close and +cancellation of a still-opening socket pass under the race detector. The Native +engine, Host, Viewer and control package suites also pass with the overlay. +Actual control-WebSocket cancellation during slow transport preparation remains +open: direct invocation and explicit session close do not cover the loopback +command queue. The experiment has no new recovery loop or room resource registry. + The earlier FFmpeg matrix had one decoded-content failure. Matched synthetic bitstream/RTP replay identifies a receiver-fixture mismatch: its separate SDK assembler emits a truncated startup frame and can resume dependent frames after diff --git a/docs/todo.md b/docs/todo.md index 8c79104b1..928a30ff4 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -94,7 +94,9 @@ Implementation needs a concrete useful result within the product model. connection material now travels through that assigned-connection owner; trusted relay configuration and event-to-control dispatch remain fixture inputs. The isolated authenticated-session capability projection and Browser schema/ - diagnostic checks pass; App key production, early discovery and transport + diagnostic checks pass. Socket binding now follows the individual Native + request and its existing media owner; complete loopback cancellation during + slow preparation, App key production, early discovery and transport consumption remain open. Integrate those boundaries at the existing Browser/App connection owners. No product wire extension or Browser integration is implemented. From ac6816f5fbecaa3b2ea65b6b4e29cbd621fcfa57 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 10:42:31 +0800 Subject: [PATCH 113/262] docs(network): record loopback cancellation evidence --- docs/research/nat-traversal.md | 13 ++++++++++--- docs/todo.md | 10 +++++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 5b76b5b8c..87142477b 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -859,9 +859,16 @@ socket owner only after input/capacity admission; reuse does not allocate again. Unsupported-codec failure, independent receiver retirement, control close and cancellation of a still-opening socket pass under the race detector. The Native engine, Host, Viewer and control package suites also pass with the overlay. -Actual control-WebSocket cancellation during slow transport preparation remains -open: direct invocation and explicit session close do not cover the loopback -command queue. The experiment has no new recovery loop or room resource registry. +The complete loopback WebSocket check exposes one additional integration gap: +delayed socket preparation waits for its media owner, while deferred session +close waits for that synchronous command. Client disconnect and server shutdown +both leave it waiting in the unmodified owner. The isolated correction runs +session close on control-context cancellation and joins it before releasing the +capacity slot. Both cases, the direct-owner checks and loopback/control package +suites pass under the race detector. This is an experimental slow-transport +requirement, not an established published failure. Browser-driven key production, +discovery and transport consumption remain open; no new recovery loop or room +resource registry is added. The earlier FFmpeg matrix had one decoded-content failure. Matched synthetic bitstream/RTP replay identifies a receiver-fixture mismatch: its separate SDK diff --git a/docs/todo.md b/docs/todo.md index 928a30ff4..b641e696e 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -94,11 +94,11 @@ Implementation needs a concrete useful result within the product model. connection material now travels through that assigned-connection owner; trusted relay configuration and event-to-control dispatch remain fixture inputs. The isolated authenticated-session capability projection and Browser schema/ - diagnostic checks pass. Socket binding now follows the individual Native - request and its existing media owner; complete loopback cancellation during - slow preparation, App key production, early discovery and transport - consumption remain open. Integrate those boundaries at the existing - Browser/App connection owners. + diagnostic checks pass. Socket binding follows the individual Native request + and its existing media owner. The isolated loopback correction cancels slow + preparation when the control ends and joins cleanup; App key production, early + discovery and transport consumption remain open. Integrate those boundaries + at the existing Browser/App connection owners. No product wire extension or Browser integration is implemented. Obtain supported direct-media and initial endpoint-discovery contracts; the isolated packet-policy overlay and bootstrap repair are not product APIs. From 068eba8b51d359a604a7a861b693b21cd46ff7a6 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 10:56:32 +0800 Subject: [PATCH 114/262] docs(network): record native handshake identity evidence --- docs/research/nat-traversal.md | 19 ++++++++++++++----- docs/todo.md | 7 ++++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 87142477b..21f408893 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -866,9 +866,9 @@ both leave it waiting in the unmodified owner. The isolated correction runs session close on control-context cancellation and joins it before releasing the capacity slot. Both cases, the direct-owner checks and loopback/control package suites pass under the race detector. This is an experimental slow-transport -requirement, not an established published failure. Browser-driven key production, -discovery and transport consumption remain open; no new recovery loop or room -resource registry is added. +requirement, not an established published failure. Viewer-page startup and +transport consumption remain open; no new recovery loop or room resource +registry is added. The earlier FFmpeg matrix had one decoded-content failure. Matched synthetic bitstream/RTP replay identifies a receiver-fixture mismatch: its separate SDK @@ -1035,8 +1035,17 @@ receive none. The existing restart request can replace or clear the key only for the current physical connection; a retired request cannot change it. Invitation revocation removes it with the authenticated session. Browser-only and Native cases pass through real room WebSockets under the race detector. -The actual App key producer and sender/receiver consumers are still unimplemented; -advertised availability is a fixture input, not a new media-readiness authority. +The isolated Native control now generates the private key in its existing +session and supplies only the public half in the ordinary `ready` handshake. +There is no persisted device identity or additional query. A Node-driven real +Browser `NativeClient` connects through loopback HTTP/WebSocket: independent +sessions have different keys, ordinary commands retain them, and retirement +clears the advertised identity without affecting its sibling. Diagnostic export +and console output exclude those keys; loopback/control race suites and the App +composition build pass. This does not cover Browser local-network permission. +The Viewer page still discovers Native on the offer rather than before room +authentication, and sender/receiver transport consumption remains unimplemented. +The key is availability metadata, not media-readiness authority. `room-browser.mjs` bundles the real Browser schemas and diagnostics with matching experimental fields. It accepts the Go fixture's twelve route/restart messages, diff --git a/docs/todo.md b/docs/todo.md index b641e696e..1e19dfd7f 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -96,9 +96,10 @@ Implementation needs a concrete useful result within the product model. The isolated authenticated-session capability projection and Browser schema/ diagnostic checks pass. Socket binding follows the individual Native request and its existing media owner. The isolated loopback correction cancels slow - preparation when the control ends and joins cleanup; App key production, early - discovery and transport consumption remain open. Integrate those boundaries - at the existing Browser/App connection owners. + preparation when the control ends and joins cleanup. App session key production + and the actual NativeClient discovery/hello pass in a Node-driven local check; + early Viewer-page discovery, reconnect updates and transport consumption remain + open. Integrate those boundaries at the existing Browser/App connection owners. No product wire extension or Browser integration is implemented. Obtain supported direct-media and initial endpoint-discovery contracts; the isolated packet-policy overlay and bootstrap repair are not product APIs. From c10561b5b999c866b49a92efe91064c2e048aadb Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:14:48 +0800 Subject: [PATCH 115/262] docs(network): record viewer startup experiment boundary --- docs/research/nat-traversal.md | 16 ++++++++++++---- docs/todo.md | 9 ++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 21f408893..ac22df87c 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1022,8 +1022,7 @@ owner keep connection material out of its output; complete protocol/signaling race suites and the direct-media composition pass. Trusted relay configuration and room-event dispatch to Native control remain fixture inputs. Complete Browser/App authorization and transport selection remain -open: the current Native-capable Viewer chooses its backend on the SDP offer, -so it does not yet advertise Native transport availability before negotiation. +open: the product Native-capable Viewer chooses its backend on the SDP offer. This check establishes packet reception and resource retirement, not decoded playback or field benefit. @@ -1043,8 +1042,17 @@ sessions have different keys, ordinary commands retain them, and retirement clears the advertised identity without affecting its sibling. Diagnostic export and console output exclude those keys; loopback/control race suites and the App composition build pass. This does not cover Browser local-network permission. -The Viewer page still discovers Native on the offer rather than before room -authentication, and sender/receiver transport consumption remains unimplemented. +The isolated Browser overlay now preflights the existing Viewer connection +promise before authentication only for App-launched pages. Missing App discovery +retains Browser fallback. The existing signaling identity supplies the current +key on authentication, restart and reconnect; control loss or disabled Native +reception clears it without initiating another recovery operation. Owner checks +cover stale cleanup, completion after unmount and disabling Native during +discovery. They exercise extracted page logic and the real signaling serializer, +not rendered UI or Browser permissions. Full overlaid frontend type checking +still fails where media consumers assume every signal is SDP/ICE. Native metadata +dispatch and sender/receiver transport consumption remain unimplemented; +the scoped startup checks do not establish complete Browser integration. The key is availability metadata, not media-readiness authority. `room-browser.mjs` bundles the real Browser schemas and diagnostics with matching diff --git a/docs/todo.md b/docs/todo.md index 1e19dfd7f..d74801bfc 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -98,9 +98,12 @@ Implementation needs a concrete useful result within the product model. and its existing media owner. The isolated loopback correction cancels slow preparation when the control ends and joins cleanup. App session key production and the actual NativeClient discovery/hello pass in a Node-driven local check; - early Viewer-page discovery, reconnect updates and transport consumption remain - open. Integrate those boundaries at the existing Browser/App connection owners. - No product wire extension or Browser integration is implemented. + the isolated Viewer startup owner now preflights App discovery and carries its + current identity through authentication/restart/reconnect. Rendered Browser + acceptance, Native metadata dispatch and transport consumption remain open; + full overlaid frontend type checking still exposes the unimplemented dispatch + boundary. Integrate these at the existing Browser/App connection owners. + No product wire extension or complete Browser integration is implemented. Obtain supported direct-media and initial endpoint-discovery contracts; the isolated packet-policy overlay and bootstrap repair are not product APIs. Carry the repaired library ordering through the complete room/App composition; From 8629bf6cba9bf00e8b0308529914aeca304db0b5 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:29:23 +0800 Subject: [PATCH 116/262] docs(network): record single-offer transport evidence --- docs/research/nat-traversal.md | 36 ++++++++++++++++------------------ docs/todo.md | 12 ++++++------ 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index ac22df87c..a29ff569c 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -986,15 +986,13 @@ dependency corrections remain. This removes the need for that extra startup API when key admission precedes media negotiation; it does not implement room-scoped key exchange or demonstrate higher direct reachability. -An isolated Go signaling extension now carries bounded bootstrap metadata through -the real room WebSocket and existing assigned-connection authorizer. It forwards -candidate and retained-connection metadata, rejects unrelated peers/rooms and -retired connection IDs, and cannot adopt a new connection from a key message. -Ordinary invitation revocation still closes the authenticated Viewer session. -The protocol/signaling race suites pass; removing the connection guard fails -the scoped probe. This establishes metadata authorization. The generated -extension adds no server key registry and does not change the product wire -format or Browser implementation. +The isolated signaling extension carries bounded listener metadata on the +assigned parent's ordinary SDP offer. SDP/ICE remain the signal variants; +there is no extra key-exchange message or router authorization branch. Actual +room WebSocket checks reject unrelated peers/rooms, retired candidates and +listener metadata on answers or ICE candidates. Ordinary invitation revocation +still closes the authenticated Viewer session. Protocol/signaling race suites +pass. The generated extension adds no server key registry or product wire change. A separate Native control check derives admission from its actual owned inbound flows. Closing one flow retains the shared peer's authorization while another @@ -1008,13 +1006,13 @@ The pending-flow adapter also rejects a different admitted peer without consumin the assigned listener. The two-peer control reproduced the former premature close; the repair permits the expected peer or cancels through the same owner. -The composed room/Native control passes actual room-authorized bootstrap exchange, -SDP/candidates and RTP delivery, same-parent replacement after DERP closure, -late old-receiver cleanup and invitation-driven final retirement. Rebuilding -uses the current assignment to exchange the key before ICE; the subsequent SDP -offer adopts the replacement ID. Trying to exchange a key on the new ID first -is rejected. Thus pre-ICE preparation does not require giving metadata the power -to adopt a connection. Listener metadata exchanges the node and discovery keys, +The composed room/Native control receives the child's authenticated key through +its assigned-parent projection. The parent prepares a listener before its offer; +the child dials from that offer before creating its ICE agent. RTP delivery, +same-parent replacement after DERP closure, late old-receiver cleanup and +invitation-driven final retirement pass under the race detector. The ordinary +offer adopts a replacement ID; the parent's updated assignment retires its +listener while the Viewer stops receiving. Listener metadata carries the node and discovery keys, a nonzero shared key and a bounded port; it carries no relay URLs or region IDs. The dialer constructs the upstream address from received fields and a trusted local relay configuration. Strict shape checks and the existing route diagnostic @@ -1049,9 +1047,9 @@ key on authentication, restart and reconnect; control loss or disabled Native reception clears it without initiating another recovery operation. Owner checks cover stale cleanup, completion after unmount and disabling Native during discovery. They exercise extracted page logic and the real signaling serializer, -not rendered UI or Browser permissions. Full overlaid frontend type checking -still fails where media consumers assume every signal is SDP/ICE. Native metadata -dispatch and sender/receiver transport consumption remain unimplemented; +not rendered UI or Browser permissions. Full overlaid frontend and lifecycle-test +type checking now pass with the single offer field. Native command binding +and sender/receiver transport consumption remain unimplemented; the scoped startup checks do not establish complete Browser integration. The key is availability metadata, not media-readiness authority. diff --git a/docs/todo.md b/docs/todo.md index d74801bfc..3cccc9c35 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -89,9 +89,9 @@ Implementation needs a concrete useful result within the product model. accepted. Resolve key exchange and admission before ICE at the assigned-connection owner; research owns the supported pre-ICE dial/retirement checks and the isolated - gated counterfactual. The Go-only room/Native composition verifies authorized - key exchange, same-parent replacement and invitation-driven retirement; - connection material now travels through that assigned-connection owner; + gated counterfactual. The Go-only room/Native composition verifies the + authenticated receiver key and listener metadata on the ordinary SDP offer, + same-parent replacement and invitation-driven retirement; trusted relay configuration and event-to-control dispatch remain fixture inputs. The isolated authenticated-session capability projection and Browser schema/ diagnostic checks pass. Socket binding follows the individual Native request @@ -100,9 +100,9 @@ Implementation needs a concrete useful result within the product model. and the actual NativeClient discovery/hello pass in a Node-driven local check; the isolated Viewer startup owner now preflights App discovery and carries its current identity through authentication/restart/reconnect. Rendered Browser - acceptance, Native metadata dispatch and transport consumption remain open; - full overlaid frontend type checking still exposes the unimplemented dispatch - boundary. Integrate these at the existing Browser/App connection owners. + acceptance, Native command binding and transport consumption remain open. + Full overlaid frontend type checking passes without extra signal variants. + Integrate the remaining work at the existing Browser/App connection owners. No product wire extension or complete Browser integration is implemented. Obtain supported direct-media and initial endpoint-discovery contracts; the isolated packet-policy overlay and bootstrap repair are not product APIs. From 046d990be1b4b37387edd2ad84be41eb1a06e6d5 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:59:33 +0800 Subject: [PATCH 117/262] docs(network): record receiver binding and identity limits --- docs/research/nat-traversal.md | 39 ++++++++++++++++----- docs/todo.md | 62 +++++++++++----------------------- 2 files changed, 50 insertions(+), 51 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index a29ff569c..5857caf72 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -866,9 +866,10 @@ both leave it waiting in the unmodified owner. The isolated correction runs session close on control-context cancellation and joins it before releasing the capacity slot. Both cases, the direct-owner checks and loopback/control package suites pass under the race detector. This is an experimental slow-transport -requirement, not an established published failure. Viewer-page startup and -transport consumption remain open; no new recovery loop or room resource -registry is added. +requirement, not an established published failure. The +[authorization boundary](#peer-authorization-boundary) owns the subsequent +Viewer startup and receive-command evidence; no new recovery loop or room +resource registry is added. The earlier FFmpeg matrix had one decoded-content failure. Matched synthetic bitstream/RTP replay identifies a receiver-fixture mismatch: its separate SDK @@ -1032,8 +1033,9 @@ receive none. The existing restart request can replace or clear the key only for the current physical connection; a retired request cannot change it. Invitation revocation removes it with the authenticated session. Browser-only and Native cases pass through real room WebSockets under the race detector. -The isolated Native control now generates the private key in its existing -session and supplies only the public half in the ordinary `ready` handshake. +The explicitly configured experimental Native control generates its private key +and supplies only the public half in the ordinary `ready` handshake. An +unconfigured control advertises no experimental transport. There is no persisted device identity or additional query. A Node-driven real Browser `NativeClient` connects through loopback HTTP/WebSocket: independent sessions have different keys, ordinary commands retain them, and retirement @@ -1047,12 +1049,31 @@ key on authentication, restart and reconnect; control loss or disabled Native reception clears it without initiating another recovery operation. Owner checks cover stale cleanup, completion after unmount and disabling Native during discovery. They exercise extracted page logic and the real signaling serializer, -not rendered UI or Browser permissions. Full overlaid frontend and lifecycle-test -type checking now pass with the single offer field. Native command binding -and sender/receiver transport consumption remain unimplemented; -the scoped startup checks do not establish complete Browser integration. +not rendered UI or Browser permissions. The overlaid NativeViewerPeer and +NativeClient forward the offer's listener to `receive-offer`; full frontend +type checking passes. The actual command now creates the Tailcat flow lazily +under its admitted Native receiver, using the control's key and locally trusted +relay. Shared strict decoding rejects null or untrusted configuration fields; +unavailable transport is an operation failure. Control retirement cancels a +real pending dial. Same-parent overlap shares one client across two physical +flows; final retirement releases it. Room/Native RTP and cancellation checks pass +under the race detector. Sender binding, rendered Browser acceptance and complete +recovery remain open. These checks do not establish complete Browser integration. The key is availability metadata, not media-readiness authority. +The independent-parent control on 2026-10-10 exposes an identity-scope limit. +It receives RTP on both local direct paths, but two parent-bound Tailcat clients +using the same control key produce two duplicate relay registrations; the +same-parent control produces none. Upstream +[DERP registration](https://github.com/tailscale/tailscale/blob/main/derp/derpserver/derpserver.go) +selects an active connection among duplicates, so local direct delivery does not +prove reliable discovery/recovery across parents. Revisit connection identity or +supported shared-underlay ownership before adoption; do not weaken overlap or +add a route-specific fallback to conceal this limit. The same test exposed a +pending-listener close race: the dependency may clear its remote address before +a final datagram write. Retaining the already-authorized address makes that +write fail normally; the focused retirement and concurrent ICE checks pass. + `room-browser.mjs` bundles the real Browser schemas and diagnostics with matching experimental fields. It accepts the Go fixture's twelve route/restart messages, rejects malformed capability/listener fields and removes scoped keys from both diff --git a/docs/todo.md b/docs/todo.md index 3cccc9c35..91e34582d 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -72,49 +72,27 @@ Implementation needs a concrete useful result within the product model. of v1.9.2. - [ ] **Native Magicsock.** Go reuse is feasible, but the [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) - have not established improved direct reachability. Seek a repeatable benefit - before accepting room-scoped peer keys, flow admission and relay operation. - Verify the measured local-address handoff benefit through the complete App, - mixed Browser/Native topology and physical networks. Current Native-engine, - Browser-bridge and decoded-picture controls do not cover App authorization or - room routing, and do not establish another reachable NAT class. Preserve the - existing local address policy; forcing loopback fails Firefox. - Product adoption still needs supported - [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary) - and RTCP/MTU boundaries, complete recovery with fresh per-connection flows - beyond the isolated owner and decoded-handoff checks, including each sender's - recovery capability, and mixed Browser/Native verification. Inner checks can - exhaust before the underlay recovers; research distinguishes eventual recovery - from exhaustion by sender behavior. No new product setting or flow owner is - accepted. - Resolve key exchange and admission before ICE at the assigned-connection owner; - research owns the supported pre-ICE dial/retirement checks and the isolated - gated counterfactual. The Go-only room/Native composition verifies the - authenticated receiver key and listener metadata on the ordinary SDP offer, - same-parent replacement and invitation-driven retirement; - trusted relay configuration and event-to-control dispatch remain fixture inputs. - The isolated authenticated-session capability projection and Browser schema/ - diagnostic checks pass. Socket binding follows the individual Native request - and its existing media owner. The isolated loopback correction cancels slow - preparation when the control ends and joins cleanup. App session key production - and the actual NativeClient discovery/hello pass in a Node-driven local check; - the isolated Viewer startup owner now preflights App discovery and carries its - current identity through authentication/restart/reconnect. Rendered Browser - acceptance, Native command binding and transport consumption remain open. - Full overlaid frontend type checking passes without extra signal variants. - Integrate the remaining work at the existing Browser/App connection owners. + have not established another reachable NAT class. Verify the measured local + address-handoff benefit through the complete App, mixed Browser/Native topology + and physical networks before adoption. Resolve the measured independent-parent + identity collision: separate Tailcat clients with the same session key register + duplicate relay connections even while local direct media passes. + Verify receiver reuse when the same connection ID receives changed listener + metadata; its existing UDP flow must not silently retain the previous listener. + Complete sender binding, room-event dispatch, rendered Browser acceptance and + recovery at the existing assigned-connection owners. Key exchange/admission + must precede ICE; diagnostic replies are not media-readiness evidence, and inner + checks may exhaust before the underlay recovers. Research owns the verified + receive-command, key/lifecycle and local RTP evidence and their limits. + Obtain supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary), + direct-media, endpoint-discovery and RTCP/MTU contracts plus a trusted relay + configuration. Experimental dependency patches are not product APIs. + Preserve the local address policy (forcing loopback fails Firefox), existing + recovery deadlines and single route/resource owners. DERP must not become + another media fallback or bypass site/share policy. No new setting, timeout + change or transport integration is accepted; the P2P evidence work below owns + network/interop acceptance. No product wire extension or complete Browser integration is implemented. - Obtain supported direct-media and initial endpoint-discovery contracts; - the isolated packet-policy overlay and bootstrap repair are not product APIs. - Carry the repaired library ordering through the complete room/App composition; - neither diagnostic replies nor isolated handoffs establish full recovery. - DERP must not silently become another media fallback or bypass site/share - route policy. - Keep isolated dependency patches and - recovery counterfactuals outside product routing; a diagnostic reply is not - media-readiness authority. Research owns the experiments and their limits. - No timeout change or transport integration is accepted; the P2P evidence work - below owns the network/interop acceptance boundary. - [ ] **Small external test before 2.0.** First use the isolated transport experiment on the same difficult network pairs as current Piik, measuring direct success separately from relay delivery, first-picture time, recovery From a705e65d842afaae33266a221176259f1a8fdce6 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:28:42 +0800 Subject: [PATCH 118/262] docs(network): record experimental receiver recovery boundary --- docs/research/nat-traversal.md | 15 ++++++++++++++- docs/todo.md | 6 ++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 5857caf72..c6f586db1 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1057,7 +1057,11 @@ relay. Shared strict decoding rejects null or untrusted configuration fields; unavailable transport is an operation failure. Control retirement cancels a real pending dial. Same-parent overlap shares one client across two physical flows; final retirement releases it. Room/Native RTP and cancellation checks pass -under the race detector. Sender binding, rendered Browser acceptance and complete +under the race detector. Receiver reuse additionally requires identical listener +metadata. The changed-port control fails without that check; with a shared +client retained by a sibling, replacement receives fresh RTP and leaves the +sibling working. Ordinary Native reuse/control race suites still pass. +Sender binding, rendered Browser acceptance and complete recovery remain open. These checks do not establish complete Browser integration. The key is availability metadata, not media-readiness authority. @@ -1074,6 +1078,15 @@ pending-listener close race: the dependency may clear its remote address before a final datagram write. Retaining the already-authorized address makes that write fail normally; the focused retirement and concurrent ICE checks pass. +Last-receiver replacement exposes a separate recovery limit: recreating the +client with the same key completes bootstrap but delivers no RTP within the +unchanged five-second window. A tunnel-only generated-UDP control reproduces +the failure without Piik or Pion; explicitly retiring the parent's peer before +recreating the client passes. Allowing local relay data or re-advertising server +endpoints on a known-key handshake does not repair it. The precise retained +dependency state remains unisolated. Keep this opt-in negative control with the +identity-scope work above; do not add a parent reset that would retire siblings. + `room-browser.mjs` bundles the real Browser schemas and diagnostics with matching experimental fields. It accepts the Go fixture's twelve route/restart messages, rejects malformed capability/listener fields and removes scoped keys from both diff --git a/docs/todo.md b/docs/todo.md index 91e34582d..5fc1e4e61 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -77,8 +77,10 @@ Implementation needs a concrete useful result within the product model. and physical networks before adoption. Resolve the measured independent-parent identity collision: separate Tailcat clients with the same session key register duplicate relay connections even while local direct media passes. - Verify receiver reuse when the same connection ID receives changed listener - metadata; its existing UDP flow must not silently retain the previous listener. + Resolve same-key client recreation while the parent retains its peer: bootstrap + succeeds but both the UDP-only control and last-receiver replacement deliver + no datagrams within their unchanged bounds. Retiring the parent peer passes + the control; that is evidence, not permission to reset healthy sibling paths. Complete sender binding, room-event dispatch, rendered Browser acceptance and recovery at the existing assigned-connection owners. Key exchange/admission must precede ICE; diagnostic replies are not media-readiness evidence, and inner From e47326bd46e909c9fadbe704f0af76372b7f8a76 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:49:38 +0800 Subject: [PATCH 119/262] docs(network): record bounded discovery recovery evidence --- docs/research/nat-traversal.md | 19 ++++++++++++++----- docs/todo.md | 7 +++---- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index c6f586db1..49a0c69b1 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1081,11 +1081,20 @@ write fail normally; the focused retirement and concurrent ICE checks pass. Last-receiver replacement exposes a separate recovery limit: recreating the client with the same key completes bootstrap but delivers no RTP within the unchanged five-second window. A tunnel-only generated-UDP control reproduces -the failure without Piik or Pion; explicitly retiring the parent's peer before -recreating the client passes. Allowing local relay data or re-advertising server -endpoints on a known-key handshake does not repair it. The precise retained -dependency state remains unisolated. Keep this opt-in negative control with the -identity-scope work above; do not add a parent reset that would retire siblings. +the failure without Piik or Pion. Keyless event counters isolate the first +WireGuard response going to the old endpoint; the next normal handshake uses +the new endpoint and delivers data after the original window. This is delayed +recovery, not permanent loss. Allowing relay data or repeating endpoint +advertisement alone does not meet the original bound. + +`tailcat-discovery-lifetime.patch` gives each new client engine a fresh discovery +key while preserving its authorized node key. The authenticated meow updates an +immutable peer snapshot, so existing Magicsock invalidation retires the cached +path. Server discovery identity stays stable for published addresses. UDP-only +and actual Native last-receiver replacement now pass without changing timeouts +or resetting the parent's peer; sibling, revocation, listener and upstream +admission checks pass under the race detector. This remains an isolated +dependency candidate, not a supported API or a fix for duplicate relay identities. `room-browser.mjs` bundles the real Browser schemas and diagnostics with matching experimental fields. It accepts the Go fixture's twelve route/restart messages, diff --git a/docs/todo.md b/docs/todo.md index 5fc1e4e61..17925307b 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -77,10 +77,9 @@ Implementation needs a concrete useful result within the product model. and physical networks before adoption. Resolve the measured independent-parent identity collision: separate Tailcat clients with the same session key register duplicate relay connections even while local direct media passes. - Resolve same-key client recreation while the parent retains its peer: bootstrap - succeeds but both the UDP-only control and last-receiver replacement deliver - no datagrams within their unchanged bounds. Retiring the parent peer passes - the control; that is evidence, not permission to reset healthy sibling paths. + Obtain supported client discovery-lifetime behavior: the isolated repair meets + the original reconnect bounds without resetting the parent peer, but remains + another dependency patch rather than a product-ready integration. Complete sender binding, room-event dispatch, rendered Browser acceptance and recovery at the existing assigned-connection owners. Key exchange/admission must precede ICE; diagnostic replies are not media-readiness evidence, and inner From 12628a5e578b96f9144fa0493fb214914010716a Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:17:42 +0800 Subject: [PATCH 120/262] docs(network): record client-scoped transport evidence --- docs/research/nat-traversal.md | 143 +++++++++++++-------------------- docs/todo.md | 11 +-- 2 files changed, 59 insertions(+), 95 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 49a0c69b1..f25493008 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1007,94 +1007,61 @@ The pending-flow adapter also rejects a different admitted peer without consumin the assigned listener. The two-peer control reproduced the former premature close; the repair permits the expected peer or cancels through the same owner. -The composed room/Native control receives the child's authenticated key through -its assigned-parent projection. The parent prepares a listener before its offer; -the child dials from that offer before creating its ICE agent. RTP delivery, -same-parent replacement after DERP closure, late old-receiver cleanup and -invitation-driven final retirement pass under the race detector. The ordinary -offer adopts a replacement ID; the parent's updated assignment retires its -listener while the Viewer stops receiving. Listener metadata carries the node and discovery keys, -a nonzero shared key and a bounded port; it carries no relay URLs or region IDs. -The dialer constructs the upstream address from received fields and a trusted -local relay configuration. Strict shape checks and the existing route diagnostic -owner keep connection material out of its output; complete protocol/signaling -race suites and the direct-media composition pass. -Trusted relay configuration and room-event dispatch to Native control remain -fixture inputs. Complete Browser/App authorization and transport selection remain -open: the product Native-capable Viewer chooses its backend on the SDP offer. -This check establishes packet reception and resource retirement, not decoded -playback or field benefit. - -An isolated session-capability candidate addresses that timing boundary without -a second exchange or room-wide key registry. Viewer authentication supplies its -public Native receiver key; the existing preparing and committed assignments -project it only to assigned parents. Public presence and unrelated participants -receive none. The existing restart request can replace or clear the key only -for the current physical connection; a retired request cannot change it. -Invitation revocation removes it with the authenticated session. Browser-only -and Native cases pass through real room WebSockets under the race detector. -The explicitly configured experimental Native control generates its private key -and supplies only the public half in the ordinary `ready` handshake. An -unconfigured control advertises no experimental transport. -There is no persisted device identity or additional query. A Node-driven real -Browser `NativeClient` connects through loopback HTTP/WebSocket: independent -sessions have different keys, ordinary commands retain them, and retirement -clears the advertised identity without affecting its sibling. Diagnostic export -and console output exclude those keys; loopback/control race suites and the App -composition build pass. This does not cover Browser local-network permission. -The isolated Browser overlay now preflights the existing Viewer connection -promise before authentication only for App-launched pages. Missing App discovery -retains Browser fallback. The existing signaling identity supplies the current -key on authentication, restart and reconnect; control loss or disabled Native -reception clears it without initiating another recovery operation. Owner checks -cover stale cleanup, completion after unmount and disabling Native during -discovery. They exercise extracted page logic and the real signaling serializer, -not rendered UI or Browser permissions. The overlaid NativeViewerPeer and -NativeClient forward the offer's listener to `receive-offer`; full frontend -type checking passes. The actual command now creates the Tailcat flow lazily -under its admitted Native receiver, using the control's key and locally trusted -relay. Shared strict decoding rejects null or untrusted configuration fields; -unavailable transport is an operation failure. Control retirement cancels a -real pending dial. Same-parent overlap shares one client across two physical -flows; final retirement releases it. Room/Native RTP and cancellation checks pass -under the race detector. Receiver reuse additionally requires identical listener -metadata. The changed-port control fails without that check; with a shared -client retained by a sibling, replacement receives fresh RTP and leaves the -sibling working. Ordinary Native reuse/control race suites still pass. -Sender binding, rendered Browser acceptance and complete -recovery remain open. These checks do not establish complete Browser integration. -The key is availability metadata, not media-readiness authority. - -The independent-parent control on 2026-10-10 exposes an identity-scope limit. -It receives RTP on both local direct paths, but two parent-bound Tailcat clients -using the same control key produce two duplicate relay registrations; the -same-parent control produces none. Upstream +The composed room/Native experiment separates session availability from actual +transport identity. Native control advertises only availability in its ordinary +ready response; authentication and existing assignments project that fact only +to assigned parents. Restart/reconnect can clear it through the same owners. +No persistent device key or room-wide key registry is added. App-launched Viewer +startup uses its existing discovery promise; missing App retains Browser fallback. +Late discovery, unmount, control replacement and stale close remain fenced. + +The parent reserves its listener and sends its bounded metadata with the ordinary +SDP offer. The actual receive command creates or reuses the parent-bound Tailcat +client and emits its public key through a connection-scoped Native event before +dialing. An experimental `native-peer` signal returns that key to the exact +assigned listener; its grant precedes receiver ICE. Parent-side binding checks +the complete listener identity, is immutable after binding, and ends with that +physical flow. Preparing and committed routes use the existing authorizer; +reverse, unrelated, cross-room and retired-connection messages are rejected. +Listener metadata accepts no relay URL or region. Trusted relay configuration +remains local, and diagnostics exclude connection keys. + +On 2026-10-10 the previous session-wide key produced duplicate DERP registrations +when two distinct parent-bound clients overlapped. Upstream [DERP registration](https://github.com/tailscale/tailscale/blob/main/derp/derpserver/derpserver.go) -selects an active connection among duplicates, so local direct delivery does not -prove reliable discovery/recovery across parents. Revisit connection identity or -supported shared-underlay ownership before adoption; do not weaken overlap or -add a route-specific fallback to conceal this limit. The same test exposed a -pending-listener close race: the dependency may clear its remote address before -a final datagram write. Retaining the already-authorized address makes that -write fail normally; the focused retirement and concurrent ICE checks pass. - -Last-receiver replacement exposes a separate recovery limit: recreating the -client with the same key completes bootstrap but delivers no RTP within the -unchanged five-second window. A tunnel-only generated-UDP control reproduces -the failure without Piik or Pion. Keyless event counters isolate the first -WireGuard response going to the old endpoint; the next normal handshake uses -the new endpoint and delivers data after the original window. This is delayed -recovery, not permanent loss. Allowing relay data or repeating endpoint -advertisement alone does not meet the original bound. - -`tailcat-discovery-lifetime.patch` gives each new client engine a fresh discovery -key while preserving its authorized node key. The authenticated meow updates an -immutable peer snapshot, so existing Magicsock invalidation retires the cached -path. Server discovery identity stays stable for published addresses. UDP-only -and actual Native last-receiver replacement now pass without changing timeouts -or resetting the parent's peer; sibling, revocation, listener and upstream -admission checks pass under the race detector. This remains an isolated -dependency candidate, not a supported API or a fix for duplicate relay identities. +selects among duplicate connections; local direct RTP had hidden this conflict. +The corrected adapter uses Tailcat's default fresh identity per client, sharing +that client only across physical flows to the same parent address. Both the +same-parent and independent-tunnel-parent controls now require zero duplicates +and fresh RTP before and after old-flow retirement. These controls use one +assigned room parent with two listener instances; they do not yet prove a full +room-controller handoff between different participant parents. + +Fresh client identity also avoids the measured same-key recreation delay: the +old first WireGuard response went to a cached endpoint and data arrived only on +the next handshake. Last-receiver replacement now meets the original five-second +media bound without resetting the parent, extending timeouts or patching client +discovery lifetime. The superseded dependency patch is removed. Receiver reuse +still requires identical listener metadata; a changed listener replaces the +source/transport through its existing owner. Sibling and ordinary reuse checks +retain their behavior. + +Control close cancels dialing and joins each pending construction before final +client close: Tailcat's close-before-initialization is not terminal. Queued +identity events lose authority with the flow. Native control, loopback and +listener retirement checks cover cancellation and ordinary cleanup; the parent +adapter also retains its already-authorized remote address so a late write fails +normally when the dependency clears its endpoint. + +The overlaid Browser NativeViewerPeer forwards the listener and returns scoped +identity events while receive-offer is pending. Wrong share/connection and +post-disposal events are ignored; there is no extra preparation timer or pending +identity registry. Node-driven discovery, serializer, schema and lifecycle checks +are separate from rendered media acceptance. Complete sender binding, room-event +dispatch, actual multi-participant handoff and rendered Browser recovery remain +open. The fixture still supplies parent binding and translates revocation into +Native stop/listener retirement. Availability, public keys and handshake replies +are not decoded-frame readiness or evidence of additional field reachability. `room-browser.mjs` bundles the real Browser schemas and diagnostics with matching experimental fields. It accepts the Go fixture's twelve route/restart messages, @@ -1108,7 +1075,7 @@ These checks establish startup-order and ownership constraints, not a product integration or a Piik field defect. The earlier candidate gate and unsupported dependency method are not accepted product mechanisms. The existing assigned-connection owner must resolve key exchange and admission -before ICE starts. Accepting arbitrary keys, passing room credentials into Native +before receiver ICE starts. Accepting arbitrary keys, passing room credentials into Native control or enlarging its timeout is not an equivalent solution. ## Native Mapping Cache And Retry Lifetimes diff --git a/docs/todo.md b/docs/todo.md index 17925307b..5238fe919 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -74,15 +74,12 @@ Implementation needs a concrete useful result within the product model. [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) have not established another reachable NAT class. Verify the measured local address-handoff benefit through the complete App, mixed Browser/Native topology - and physical networks before adoption. Resolve the measured independent-parent - identity collision: separate Tailcat clients with the same session key register - duplicate relay connections even while local direct media passes. - Obtain supported client discovery-lifetime behavior: the isolated repair meets - the original reconnect bounds without resetting the parent peer, but remains - another dependency patch rather than a product-ready integration. + and physical networks before adoption. Extend the isolated client-identity and + listener-binding checks to actual room handoffs between different participants; + the current two-listener fixture does not establish that complete lifecycle. Complete sender binding, room-event dispatch, rendered Browser acceptance and recovery at the existing assigned-connection owners. Key exchange/admission - must precede ICE; diagnostic replies are not media-readiness evidence, and inner + must precede receiver ICE; diagnostic replies are not media-readiness evidence, and inner checks may exhaust before the underlay recovers. Research owns the verified receive-command, key/lifecycle and local RTP evidence and their limits. Obtain supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary), From 92f32389f9d70c80db8fa6a22c74555b47d7bace Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:28:24 +0800 Subject: [PATCH 121/262] docs(network): record native sender ownership checks --- docs/research/nat-traversal.md | 22 ++++++++++++++++++---- docs/todo.md | 4 ++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index f25493008..b3c683b04 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1053,14 +1053,28 @@ listener retirement checks cover cancellation and ordinary cleanup; the parent adapter also retains its already-authorized remote address so a late write fails normally when the dependency clears its endpoint. +The isolated Native sender now prepares its listener through the existing +Host/Viewer edge socket factory. Its offer returns metadata, and a strict +connection-scoped control command binds the reply through that same edge. +There is no additional control-level listener registry. The raw transport checks +and Native overlay compile the same admission/socket adapters. Closing an edge +revokes its listener; retiring the source releases its remaining children. +`TestNativeSenderOwnsListener` carries a generated VP8 source through the actual +Native receive/relay commands to two Native children. It checks fresh sibling +RTP after one child retires, idempotent replies and rejection of another listener +or a retired connection. Host/media-edge/Viewer/control race suites pass. This +fixture still pumps events instead of running the Browser route owners; it is +not physical Host capture or complete room-controller acceptance. + The overlaid Browser NativeViewerPeer forwards the listener and returns scoped identity events while receive-offer is pending. Wrong share/connection and post-disposal events are ignored; there is no extra preparation timer or pending identity registry. Node-driven discovery, serializer, schema and lifecycle checks -are separate from rendered media acceptance. Complete sender binding, room-event -dispatch, actual multi-participant handoff and rendered Browser recovery remain -open. The fixture still supplies parent binding and translates revocation into -Native stop/listener retirement. Availability, public keys and handshake replies +are separate from rendered media acceptance. Browser sender selection/binding, +room-event dispatch, actual multi-participant handoff and rendered Browser +recovery remain open. The composed room fixture still supplies parent binding +and translates revocation into Native stop/listener retirement. Availability, +public keys and handshake replies are not decoded-frame readiness or evidence of additional field reachability. `room-browser.mjs` bundles the real Browser schemas and diagnostics with matching diff --git a/docs/todo.md b/docs/todo.md index 5238fe919..09a22abee 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -77,11 +77,11 @@ Implementation needs a concrete useful result within the product model. and physical networks before adoption. Extend the isolated client-identity and listener-binding checks to actual room handoffs between different participants; the current two-listener fixture does not establish that complete lifecycle. - Complete sender binding, room-event dispatch, rendered Browser acceptance and + Complete Browser sender selection/binding, room-event dispatch, rendered acceptance and recovery at the existing assigned-connection owners. Key exchange/admission must precede receiver ICE; diagnostic replies are not media-readiness evidence, and inner checks may exhaust before the underlay recovers. Research owns the verified - receive-command, key/lifecycle and local RTP evidence and their limits. + sender/receiver command, key/lifecycle and local RTP evidence and their limits. Obtain supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary), direct-media, endpoint-discovery and RTCP/MTU contracts plus a trusted relay configuration. Experimental dependency patches are not product APIs. From a5efc2c7328d4eb29a1abfdbb62e15275efa57ad Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:02:21 +0800 Subject: [PATCH 122/262] docs(network): record rendered mixed-route acceptance --- docs/research/nat-traversal.md | 32 +++++++++++++++++++++++++------- docs/todo.md | 5 ++++- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index b3c683b04..7dabf6fa5 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1069,13 +1069,31 @@ not physical Host capture or complete room-controller acceptance. The overlaid Browser NativeViewerPeer forwards the listener and returns scoped identity events while receive-offer is pending. Wrong share/connection and post-disposal events are ignored; there is no extra preparation timer or pending -identity registry. Node-driven discovery, serializer, schema and lifecycle checks -are separate from rendered media acceptance. Browser sender selection/binding, -room-event dispatch, actual multi-participant handoff and rendered Browser -recovery remain open. The composed room fixture still supplies parent binding -and translates revocation into Native stop/listener retirement. Availability, -public keys and handshake replies -are not decoded-frame readiness or evidence of additional field reachability. +identity registry. The sender reads capability from the current preparation, +assignment or recovery message, without a second capability cache. Existing +Browser owners dispatch the reply to the listener's Native edge. Focused tests +cover stale replies, deep-copied assignment capability and recovery after a +capability change. + +An owned Linux/Chromium 152 run exercises the actual UI and room controller with +a generated 640×360 VP8 canvas source and one outbound copy per participant: +Browser Host → Native Viewer → Native Viewer → Browser Viewer. The inner Native +pair uses Tailcat; the outer Browser links retain ordinary WebRTC. Reconnecting +the second Native Viewer replaces its listener/client without accumulating +resources. After the first Native Viewer leaves, the next Viewer changes to the +Browser Host and both remaining Viewers resume decoded, changing pixels. Removing +the second Native Viewer then restores an ordinary Browser-to-Browser route. +All transport counters are zero at retirement. This is bounded local rendered +acceptance, not physical capture, difficult-NAT reachability, sustained recovery +or a handoff between two different Native parents. + +Composition also exposed duplicate address discovery: the shared candidate +collector still sent ordinary STUN probes through the injected virtual socket. +The experimental socket boundary now leaves outer discovery to Tailcat. A local +STUN control fails before this correction and passes after it; ordinary Native +gathering still produces its observed candidate. There is no timeout extension +or additional retry owner. Availability, public keys and handshake replies +remain distinct from decoded-frame readiness and additional field reachability. `room-browser.mjs` bundles the real Browser schemas and diagnostics with matching experimental fields. It accepts the Go fixture's twelve route/restart messages, diff --git a/docs/todo.md b/docs/todo.md index 09a22abee..cab427fd8 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -90,7 +90,10 @@ Implementation needs a concrete useful result within the product model. another media fallback or bypass site/share policy. No new setting, timeout change or transport integration is accepted; the P2P evidence work below owns network/interop acceptance. - No product wire extension or complete Browser integration is implemented. + Rendered local Browser/Native chains now cover reconnect and upstream departure; + the assessment owns the limits. Different Native-parent handoffs, sustained and + difficult-network recovery remain open. No product transport integration is + accepted; all experimental wire changes remain isolated. - [ ] **Small external test before 2.0.** First use the isolated transport experiment on the same difficult network pairs as current Piik, measuring direct success separately from relay delivery, first-picture time, recovery From 0c6e41cfee92b06467507bdc43a626c6274df907 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:19:28 +0800 Subject: [PATCH 123/262] docs(network): reconcile native handoff acceptance --- docs/research/nat-traversal.md | 18 +++++++++++++++--- docs/todo.md | 20 ++++++++------------ 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 7dabf6fa5..b5456cc48 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1083,9 +1083,21 @@ the second Native Viewer replaces its listener/client without accumulating resources. After the first Native Viewer leaves, the next Viewer changes to the Browser Host and both remaining Viewers resume decoded, changing pixels. Removing the second Native Viewer then restores an ordinary Browser-to-Browser route. -All transport counters are zero at retirement. This is bounded local rendered -acceptance, not physical capture, difficult-NAT reachability, sustained recovery -or a handoff between two different Native parents. +All transport counters are zero at retirement. A second rendered run uses H264 +and three Native Viewers across two App instances. When the middle Native Viewer +leaves, the last Viewer changes to the first Native Viewer, receives a new scoped +connection and resumes decoded, changing pixels; the departed participant retains +no transport resources. The normal App control-session and origin limits remain. +This is bounded local rendered acceptance, not physical capture, difficult-NAT +reachability, sustained recovery or a healthy overlapping-parent handoff. + +Cold receiver preparation initially took about three seconds in this fixture. +The local DERP region had no STUN service: two preauthorized registration controls +took about four seconds without STUN and 15–16 ms with it. Reusing Piik's STUN +server in the UI fixture reduced cold receive-offer responses to 21–25 ms. Keep +unavailable-STUN controls separate from healthy startup measurements. This finding +changes the test environment, not product timeouts or retry behavior; it does not +predict public-network latency. Composition also exposed duplicate address discovery: the shared candidate collector still sent ordinary STUN probes through the injected virtual socket. diff --git a/docs/todo.md b/docs/todo.md index cab427fd8..bb4919cd4 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -73,12 +73,11 @@ Implementation needs a concrete useful result within the product model. - [ ] **Native Magicsock.** Go reuse is feasible, but the [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) have not established another reachable NAT class. Verify the measured local - address-handoff benefit through the complete App, mixed Browser/Native topology - and physical networks before adoption. Extend the isolated client-identity and - listener-binding checks to actual room handoffs between different participants; - the current two-listener fixture does not establish that complete lifecycle. - Complete Browser sender selection/binding, room-event dispatch, rendered acceptance and - recovery at the existing assigned-connection owners. Key exchange/admission + address-handoff benefit through the complete App and physical networks before + adoption. Bounded rendered Browser/Native chains cover reconnect and departure + with a different Native parent; healthy overlapping-parent handoffs, sustained + and difficult-network recovery remain open. Preserve the existing Browser + sender, room-event and assigned-connection owners. Key exchange/admission must precede receiver ICE; diagnostic replies are not media-readiness evidence, and inner checks may exhaust before the underlay recovers. Research owns the verified sender/receiver command, key/lifecycle and local RTP evidence and their limits. @@ -88,12 +87,9 @@ Implementation needs a concrete useful result within the product model. Preserve the local address policy (forcing loopback fails Firefox), existing recovery deadlines and single route/resource owners. DERP must not become another media fallback or bypass site/share policy. No new setting, timeout - change or transport integration is accepted; the P2P evidence work below owns - network/interop acceptance. - Rendered local Browser/Native chains now cover reconnect and upstream departure; - the assessment owns the limits. Different Native-parent handoffs, sustained and - difficult-network recovery remain open. No product transport integration is - accepted; all experimental wire changes remain isolated. + change or product transport integration is accepted; all experimental wire + changes remain isolated. The P2P evidence work below owns network/interop + acceptance. - [ ] **Small external test before 2.0.** First use the isolated transport experiment on the same difficult network pairs as current Piik, measuring direct success separately from relay delivery, first-picture time, recovery From d5c812a8d266bc43f2821830cc393479ba5cd03b Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:30:47 +0800 Subject: [PATCH 124/262] docs(network): record invitation retirement acceptance --- docs/research/nat-traversal.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index b5456cc48..c811c3691 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1088,6 +1088,10 @@ and three Native Viewers across two App instances. When the middle Native Viewer leaves, the last Viewer changes to the first Native Viewer, receives a new scoped connection and resumes decoded, changing pixels; the departed participant retains no transport resources. The normal App control-session and origin limits remain. +In the mixed Browser/Native chain, revoking a private room's invitation removes +all Viewers and Native transport resources while the Host continues sharing. +The old invitation remains rejected after rotation; a fresh invitation restores +the chain and decoded media without retaining the retired transports. This is bounded local rendered acceptance, not physical capture, difficult-NAT reachability, sustained recovery or a healthy overlapping-parent handoff. From d55de819dd84dcc7cfcf18375ef7752c5509f7df Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:44:24 +0800 Subject: [PATCH 125/262] docs(network): record overlapping parent acceptance --- docs/research/nat-traversal.md | 10 +++++++++- docs/todo.md | 6 +++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index c811c3691..540c318c8 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1092,8 +1092,16 @@ In the mixed Browser/Native chain, revoking a private room's invitation removes all Viewers and Native transport resources while the Host continues sharing. The old invitation remains rejected after rotation; a fresh invitation restores the chain and decoded media without retaining the retired transports. +With copy capacity two, four Native Viewers exercise the overlapping path: +the Browser Host feeds Native A/B, and A initially feeds C/D. Controlled +relay-capacity messages move D from A to B and back through the real controller. +Holding D's first-frame acknowledgement leaves two Native client/flow pairs +alive while its old displayed picture advances; releasing it commits the new +parent and retires only the old pair. C keeps its connection and continues +decoding. Final resource and race checks pass. This exercises transport ownership +under a capacity change, not the quality-degradation trigger or its thresholds. This is bounded local rendered acceptance, not physical capture, difficult-NAT -reachability, sustained recovery or a healthy overlapping-parent handoff. +reachability or sustained recovery. Cold receiver preparation initially took about three seconds in this fixture. The local DERP region had no STUN service: two preauthorized registration controls diff --git a/docs/todo.md b/docs/todo.md index bb4919cd4..3b393f431 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -74,9 +74,9 @@ Implementation needs a concrete useful result within the product model. [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) have not established another reachable NAT class. Verify the measured local address-handoff benefit through the complete App and physical networks before - adoption. Bounded rendered Browser/Native chains cover reconnect and departure - with a different Native parent; healthy overlapping-parent handoffs, sustained - and difficult-network recovery remain open. Preserve the existing Browser + adoption. Bounded rendered Browser/Native chains cover reconnect, departure and + overlapping Native-parent changes. Sustained and difficult-network recovery + remain open. Preserve the existing Browser sender, room-event and assigned-connection owners. Key exchange/admission must precede receiver ICE; diagnostic replies are not media-readiness evidence, and inner checks may exhaust before the underlay recovers. Research owns the verified From fc54af8aec5f9d7d8de8cd600b4445dc4cdfd598 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:06:08 +0800 Subject: [PATCH 126/262] fix(media): retain source discovery during subscription --- docs/research/node-local-media-design.md | 9 ++++ docs/todo.md | 4 -- internal/app/mediaedge/quality_test.go | 7 ++- internal/media/forwarding/source.go | 13 +++++ internal/media/forwarding/source_test.go | 62 ++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 6 deletions(-) diff --git a/docs/research/node-local-media-design.md b/docs/research/node-local-media-design.md index 18589d465..2f8db494f 100644 --- a/docs/research/node-local-media-design.md +++ b/docs/research/node-local-media-design.md @@ -212,6 +212,15 @@ The replaced local Group/ Selection and receiving-PC sender-estimator registry are deleted, not retained as alternate implementations. There is no mirrored counter summing unused layers. +Source subscription must retain layer discovery that happens during registration. +LiveKit's `ReceiverBase.AddDownTrack` delivers initial layer metadata before +registering the subscriber, leaving a notification gap. The shared Source +refreshes that metadata after successful registration; DownTrack keeps its +highest observed layers, so a concurrent newer notification cannot be undone. +This boundary serves both Native outputs and SFU publication. A controlled +discovery in that gap reproduces the failure without retries or network loss; +it does not establish the cause of unmatched field reports. + The parent reads child state at actual capture InputBegin. A controlled layer switch uses the existing DownTrack resync before outputs for that input, retaining its target; this prevents an old-layer packet winning the same presentation time. diff --git a/docs/todo.md b/docs/todo.md index 3b393f431..87e7b69ac 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -16,10 +16,6 @@ Git/PRs own completed history. Prepare one coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. - Isolate the intermittent Linux race-suite no-output result in - `TestQualityDimensionsFollowAssignedSource`; focused repetitions and the next - full package run pass. No product cause is established; preserve its assertion - and timeout while collecting a matched failure. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance are complete. Integrate through the next owner-approved publishing merge and verify the separate workflow with its original CI artifacts. Mirror failure diff --git a/internal/app/mediaedge/quality_test.go b/internal/app/mediaedge/quality_test.go index 7e225ade1..3a62e7e71 100644 --- a/internal/app/mediaedge/quality_test.go +++ b/internal/app/mediaedge/quality_test.go @@ -53,8 +53,11 @@ func TestQualityDimensionsFollowAssignedSource(t *testing.T) { } time.Sleep(time.Second / 30) } - _, _, format := edge.videoCounters() - t.Fatalf("quality dimensions still refer to original source: %dx%d", uint32(format>>32), uint32(format)) + frames, _, format := edge.videoCounters() + available, rates := assigned.GetLayeredBitrate() + t.Fatalf("assigned quality: %dx%d, frames=%d, source=%t, state=%+v, layers=%v, rates=%v", + uint32(format>>32), uint32(format), frames, edge.transport.CurrentSource() == assigned.Source, + edge.transport.Output.State(), available, rates) } func TestQualitySampleUsesEstimatorCapacityAgainstEncodedPayload(t *testing.T) { diff --git a/internal/media/forwarding/source.go b/internal/media/forwarding/source.go index c4037299a..a1b553be5 100644 --- a/internal/media/forwarding/source.go +++ b/internal/media/forwarding/source.go @@ -96,6 +96,19 @@ func (source *Source) TrackInfo() *livekit.TrackInfo { return utils.CloneProto(source.currentTrackInfo.Load()) } +// ReceiverBase reads initial metadata before registering the subscriber. A +// concurrent discovery notification can fall into that gap. Refresh after +// registration; DownTrack's seen-layer setters retain the highest observation. +func (source *Source) AddDownTrack(track sfu.TrackSender) error { + if err := source.ReceiverBase.AddDownTrack(track); err != nil { + return err + } + tracker := source.StreamTrackerManager() + track.UpTrackMaxPublishedLayerChange(tracker.GetMaxPublishedLayer()) + track.UpTrackMaxTemporalLayerSeenChange(tracker.GetMaxTemporalLayerSeen()) + return nil +} + func canonicalHeaderExtensions(extensions []webrtc.RTPHeaderExtensionParameter) []webrtc.RTPHeaderExtensionParameter { result := slices.Clone(extensions) slices.SortFunc(result, func(left, right webrtc.RTPHeaderExtensionParameter) int { diff --git a/internal/media/forwarding/source_test.go b/internal/media/forwarding/source_test.go index e7a24002f..a3123ddcd 100644 --- a/internal/media/forwarding/source_test.go +++ b/internal/media/forwarding/source_test.go @@ -2,6 +2,7 @@ package forwarding import ( "errors" + "sync/atomic" "testing" "time" @@ -14,6 +15,67 @@ import ( "github.com/pion/webrtc/v4" ) +type subscribingSourceTrack struct { + *Output + publish func() + spatial atomic.Int32 +} + +func (track *subscribingSourceTrack) UpTrackMaxPublishedLayerChange(layer int32) { + if layer < 0 { + track.publish() + } + track.Output.UpTrackMaxPublishedLayerChange(layer) + track.spatial.Store(layer) +} + +type observedSourceLayers struct { + sfu.StreamTrackerManagerListener + published chan struct{} +} + +func (observer *observedSourceLayers) OnMaxPublishedLayerChanged(layer int32) { + observer.StreamTrackerManagerListener.OnMaxPublishedLayerChanged(layer) + observer.published <- struct{}{} +} + +func TestSourceSubscriptionRetainsConcurrentLayerDiscovery(t *testing.T) { + source, err := NewSource(SourceOptions{ID: "screen", StreamID: "subscription", + Codec: webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, ClockRate: 90_000, + }, PayloadType: 96}, + Formats: []LayerFormat{{Width: 8, Height: 8, Bitrate: 300_000}}, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(source.Close) + transport, err := NewTransport(TransportOptions{Source: source, ConnectionID: "joining", InitialBitrate: 1_000_000}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = transport.Close() }) + observed := &observedSourceLayers{StreamTrackerManagerListener: source.ReceiverBase, published: make(chan struct{}, 1)} + source.StreamTrackerManager().SetListener(observed) + track := &subscribingSourceTrack{Output: transport.Output} + track.publish = func() { + // Discover a layer after reading the initial snapshot but before the + // subscriber is registered. Complete its real notification in that gap. + source.StreamTrackerManager().AddTracker(0) + select { + case <-observed.published: + case <-time.After(time.Second): + t.Fatal("source did not publish its discovered layer") + } + } + if err = source.AddDownTrack(track); err != nil { + t.Fatal(err) + } + if track.spatial.Load() != source.StreamTrackerManager().GetMaxPublishedLayer() { + t.Fatalf("subscriber retained the pre-discovery snapshot: %d", track.spatial.Load()) + } +} + func TestSourceLayerBindingFeedbackAndClose(t *testing.T) { codec := webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{ MimeType: webrtc.MimeTypeVP8, ClockRate: 90_000, From 011437f76ecd2e41aa5edaf2f471bea6bdd6b391 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:15:20 +0800 Subject: [PATCH 127/262] docs(media): record bounded receiver acceptance --- docs/research/browser-screen-audio-quality.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/research/browser-screen-audio-quality.md b/docs/research/browser-screen-audio-quality.md index 0591bdea5..23cefb767 100644 --- a/docs/research/browser-screen-audio-quality.md +++ b/docs/research/browser-screen-audio-quality.md @@ -103,12 +103,17 @@ live 320/96 kbps changes retained the same subscription, with advancing decoded audio energy and video frames. Video profile changes, reload, recovery and retirement also passed; the Browser run measured stereo separation. An ordinary Browser relay forwarded received tracks to two children at 320 kbps, with -advancing decoded audio and successful cleanup. A direct Browser control now -keeps requested 256/320 kbps below an explicit 192 kbps receive limit, while a -510 kbps receiver permits both rates. Native and SFU adapter controls reject -an insufficient 192 kbps answer before binding it; retiring or correcting that -preparation leaves the original source and healthy sibling delivering media. -These controls do not establish the complete room-route response to rejection. +advancing decoded audio and successful cleanup. Direct Browser controls keep +requested 256/320 kbps below an explicit 192 kbps receive limit, while a +510 kbps receiver permits both rates. The full Browser room also retains both +receiving tracks and advancing decoded audio/video through 320/96/320 kbps edits; +only the limited child's sender is capped at 192 kbps. +Native and SFU adapter controls reject an insufficient answer before binding it. +In a full-UI SFU-only room, that rejection exhausts only the limited Viewer's +route and retires its connection. The Host, healthy sibling and subsequent +capable Viewer continue; the service Debug event retains the receive-limit +cause. Stop retires the remaining media. These are controlled SDP-limit checks, +not physical low-capability decoder measurements. A native real-Opus control through 6 Mbps -> 500 kbps -> 6 Mbps shaping retains decoded audio during live 320/96/320 kbps changes, with sustained high video after the shared From bc7762e6daf6e62a8f9c6a57eb83029441d8fcc5 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:37:25 +0800 Subject: [PATCH 128/262] docs(network): clarify optional transport adoption boundary --- docs/research/nat-traversal.md | 20 ++++++++++++++++++++ docs/todo.md | 2 ++ 2 files changed, 22 insertions(+) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 540c318c8..b37bd5583 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1119,6 +1119,26 @@ gathering still produces its observed candidate. There is no timeout extension or additional retry owner. Availability, public keys and handshake replies remain distinct from decoded-frame readiness and additional field reachability. +A separate optional-path check uses Pion's public +[`NewMultiUDPMuxDefault`](https://github.com/pion/ice/blob/v4.4.0/udp_mux_multi.go) +to advertise ordinary UDP and one prepared Tailcat flow on the same +PeerConnection. Fresh audio/video passes with either path unavailable, and +normal ICE restart selects the remaining path in both directions. Making the +unused tunnel available or closing it preserves the ordinary session and its +ICE credentials. Windows and Linux race checks pass without an application +path selector or changed ICE budgets. These local tests permit DERP and prepare +the tunnel before ICE; they establish composition, not direct-media policy, +NAT gain or cold-bootstrap independence. + +The current Native overlay still replaces its ordinary socket with the tunnel. +`DialUDPPort` waits for remote admission, while Pion's stock multi-mux requires +the sockets' addresses at construction. Simply combining them synchronously +would let unavailable bootstrap delay otherwise usable ordinary ICE. A supported +way to prepare an optional socket independently remains an adoption requirement; +the earlier private `PrepareUDPPort` patch is not a supported upstream API. +Do not replace this gap with two racing PeerConnections, a second recovery +controller or a larger product timeout. + `room-browser.mjs` bundles the real Browser schemas and diagnostics with matching experimental fields. It accepts the Go fixture's twelve route/restart messages, rejects malformed capability/listener fields and removes scoped keys from both diff --git a/docs/todo.md b/docs/todo.md index 87e7b69ac..b12211f03 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -80,6 +80,8 @@ Implementation needs a concrete useful result within the product model. Obtain supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary), direct-media, endpoint-discovery and RTCP/MTU contracts plus a trusted relay configuration. Experimental dependency patches are not product APIs. + Preserve ordinary ICE when optional bootstrap is unavailable; the current + tunnel-only Native overlay does not yet satisfy that integration boundary. Preserve the local address policy (forcing loopback fails Firefox), existing recovery deadlines and single route/resource owners. DERP must not become another media fallback or bypass site/share policy. No new setting, timeout From 41d34fd8c6f8b5a0e6cf86705af03b9d19c83a70 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:27:47 +0800 Subject: [PATCH 129/262] docs(network): record optional native composition and remaining gates --- docs/research/nat-traversal.md | 43 ++++++++++++++++++++++++++++++---- docs/todo.md | 10 ++++---- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index b37bd5583..ba5c994b9 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1130,7 +1130,7 @@ path selector or changed ICE budgets. These local tests permit DERP and prepare the tunnel before ICE; they establish composition, not direct-media policy, NAT gain or cold-bootstrap independence. -The current Native overlay still replaces its ordinary socket with the tunnel. +The default Native overlay replaces its ordinary socket with the tunnel. `DialUDPPort` waits for remote admission, while Pion's stock multi-mux requires the sockets' addresses at construction. Simply combining them synchronously would let unavailable bootstrap delay otherwise usable ordinary ICE. A supported @@ -1139,6 +1139,40 @@ the earlier private `PrepareUDPPort` patch is not a supported upstream API. Do not replace this gap with two racing PeerConnections, a second recovery controller or a larger product timeout. +An isolated Linux race control combines that reservation patch with the same +multi-mux: ordinary SRTP starts while remote admission is denied or bootstrap is +unavailable, and continues through a delayed grant. For a tunnel-only pair, +publishing its candidate only after admission establishes fresh audio/video in +the same ICE generation. Ordinary candidates are available immediately; +end-of-candidates follows the optional producer. Publishing the tunnel candidate +early instead leaves both peers connecting with no media in the matched control. +The candidate ordering uses [Trickle ICE](https://www.rfc-editor.org/rfc/rfc8838.html), +without another PeerConnection, timeout extension or application retry loop. +After ordinary ICE has nominated a pair, a late optional candidate is omitted; +it may participate in a subsequent gathering. This raw fixture permits DERP and +edits its own SDP; it does not establish direct-media policy or greater NAT coverage. + +The separate optional Native overlay now preserves the ordinary socket's STUN +and mapping owner and adds its virtual socket to that same Pion connection. +The receiver's existing candidate collector owns delayed advertisement, EOC and +cancellation; its SDP answer cannot bypass that collector. The existing flow +owner joins admission work on close. Native controls establish ordinary media +before the tunnel grant and with bootstrap unavailable. A tunnel-only control +establishes media after delayed admission, then preserves a sibling and a new +receiver through old cleanup. Candidate events continue during the delay: +bypassing the readiness gate leaves the sender checking with no media. Linux +race checks pass. These are bounded Native control/media checks, not complete +rendered App recovery, direct-only policy or physical-network acceptance. + +This composition also exposes an upstream address-lookup mismatch: Pion's +multi-mux enumerates current child addresses but `GetConn` only consults its +construction-time table. A newly enumerated address therefore has no mux. The +isolated correction resolves uncached addresses against those same child muxes, +without another address cache or socket owner; the unchanged control fails. +This preserves the ordinary socket's fresh-address contract in the experiment. +The copied dependency correction and socket-reservation method remain adoption +requirements, not supported APIs or changes to the published Piik dependency. + `room-browser.mjs` bundles the real Browser schemas and diagnostics with matching experimental fields. It accepts the Go fixture's twelve route/restart messages, rejects malformed capability/listener fields and removes scoped keys from both @@ -1150,9 +1184,10 @@ The product wire and diagnostic implementation remain unchanged. These checks establish startup-order and ownership constraints, not a product integration or a Piik field defect. The earlier candidate gate and unsupported dependency method are not accepted product mechanisms. -The existing assigned-connection owner must resolve key exchange and admission -before receiver ICE starts. Accepting arbitrary keys, passing room credentials into Native -control or enlarging its timeout is not an equivalent solution. +The existing assigned-connection and candidate owners must order admission and +advertisement so the usable tunnel receives live ICE checks; ordinary ICE need +not wait for that optional producer. Accepting arbitrary keys, passing room credentials +into Native control or enlarging its timeout is not an equivalent solution. ## Native Mapping Cache And Retry Lifetimes diff --git a/docs/todo.md b/docs/todo.md index b12211f03..efe63f8e9 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -73,15 +73,17 @@ Implementation needs a concrete useful result within the product model. adoption. Bounded rendered Browser/Native chains cover reconnect, departure and overlapping Native-parent changes. Sustained and difficult-network recovery remain open. Preserve the existing Browser - sender, room-event and assigned-connection owners. Key exchange/admission - must precede receiver ICE; diagnostic replies are not media-readiness evidence, and inner + sender, room-event and assigned-connection owners. Order admission and candidate + delivery without exhausting inner checks; diagnostic replies are not media-readiness evidence, and inner checks may exhaust before the underlay recovers. Research owns the verified sender/receiver command, key/lifecycle and local RTP evidence and their limits. Obtain supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary), direct-media, endpoint-discovery and RTCP/MTU contracts plus a trusted relay configuration. Experimental dependency patches are not product APIs. - Preserve ordinary ICE when optional bootstrap is unavailable; the current - tunnel-only Native overlay does not yet satisfy that integration boundary. + Complete optional-path App and mixed-route acceptance while preserving ordinary + ICE when bootstrap is unavailable. The bounded Native composition still needs + a private socket-reservation API and a multi-mux address-lookup correction; + verify physical interface changes and direct-only policy before adoption. Preserve the local address policy (forcing loopback fails Firefox), existing recovery deadlines and single route/resource owners. DERP must not become another media fallback or bypass site/share policy. No new setting, timeout From 96f06f9d9be9cd18428ddc24a52df383ca9c8b63 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:50:46 +0800 Subject: [PATCH 130/262] docs(network): reconcile optional recovery evidence --- docs/research/nat-traversal.md | 21 ++++++++++++++++++--- docs/todo.md | 5 +++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index ba5c994b9..147102b9e 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1161,15 +1161,30 @@ before the tunnel grant and with bootstrap unavailable. A tunnel-only control establishes media after delayed admission, then preserves a sibling and a new receiver through old cleanup. Candidate events continue during the delay: bypassing the readiness gate leaves the sender checking with no media. Linux -race checks pass. These are bounded Native control/media checks, not complete -rendered App recovery, direct-only policy or physical-network acceptance. +race checks pass. + +A bounded Chromium 152 UI check also runs the optional composition through a +Browser Host, two Native relays and a Browser Viewer. Changing decoded pixels +advance at every hop, after the second relay's manual reconnect, and after each +Native relay departs. Departure recovery respects the existing signaling grace; +retired Native transport counts reach zero. Host stop projects waiting for a +share, and the race-enabled fixture closes cleanly. This uses a synthetic source +on one local network; ordinary ICE may win. It establishes coexistence and these +recovery paths, not a tunnel-success rate, physical capture or complete App +acceptance. Optional-path invitation revocation and overlapping-parent checks +remain separate from the earlier tunnel-only UI evidence. This composition also exposes an upstream address-lookup mismatch: Pion's multi-mux enumerates current child addresses but `GetConn` only consults its construction-time table. A newly enumerated address therefore has no mux. The isolated correction resolves uncached addresses against those same child muxes, without another address cache or socket owner; the unchanged control fails. -This preserves the ordinary socket's fresh-address contract in the experiment. +A private Linux network namespace also replaces the real interface address +while retaining the sockets and PeerConnections. ICE restart gathers only the +new address and restores fresh phase-tagged audio/video. Removing only the +multi-mux correction loses that candidate in the matched control. The race +check preserves ordinary discovery; it does not exercise physical Wi-Fi handoff +or prove the optional tunnel's recovery benefit. The copied dependency correction and socket-reservation method remain adoption requirements, not supported APIs or changes to the published Piik dependency. diff --git a/docs/todo.md b/docs/todo.md index efe63f8e9..31d8d1523 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -80,8 +80,9 @@ Implementation needs a concrete useful result within the product model. Obtain supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary), direct-media, endpoint-discovery and RTCP/MTU contracts plus a trusted relay configuration. Experimental dependency patches are not product APIs. - Complete optional-path App and mixed-route acceptance while preserving ordinary - ICE when bootstrap is unavailable. The bounded Native composition still needs + Extend optional-path acceptance to invitation revocation and overlapping-parent + transitions; bounded rendered chain/reconnect/departure checks pass. Preserve + ordinary ICE when bootstrap is unavailable. The Native composition still needs a private socket-reservation API and a multi-mux address-lookup correction; verify physical interface changes and direct-only policy before adoption. Preserve the local address policy (forcing loopback fails Firefox), existing From ed7118a8bbad212ba82382a1d1397290c05837e2 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:12:35 +0800 Subject: [PATCH 131/262] fix(viewer): release native control when room access ends --- docs/research/nat-traversal.md | 12 ++-- docs/research/native-client-lifecycle.md | 13 ++++ docs/standards/presentation-lifecycle.md | 3 +- docs/todo.md | 4 +- src/client/pages/ViewerPage.tsx | 74 +++++++------------- tests/viewer-native-client-ownership.test.ts | 56 +++++++++++++-- tests/viewer-route-health.test.ts | 2 +- 7 files changed, 100 insertions(+), 64 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 147102b9e..c358e1397 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1171,8 +1171,12 @@ retired Native transport counts reach zero. Host stop projects waiting for a share, and the race-enabled fixture closes cleanly. This uses a synthetic source on one local network; ordinary ICE may win. It establishes coexistence and these recovery paths, not a tunnel-success rate, physical capture or complete App -acceptance. Optional-path invitation revocation and overlapping-parent checks -remain separate from the earlier tunnel-only UI evidence. +acceptance. Optional invitation revocation also rejects old links, clears media +resources and permits new Native reception across four grant generations. This +check exposed and now covers the shared Viewer's +[control-retirement gap](./native-client-lifecycle.md#viewer-control-retirement). +Optional overlapping-parent checks remain separate from the earlier tunnel-only +UI evidence. This composition also exposes an upstream address-lookup mismatch: Pion's multi-mux enumerates current child addresses but `GetConn` only consults its @@ -1196,8 +1200,8 @@ evidence. The Go and Browser optional-map null checks agree. This is a Node-run producer/consumer check, not Browser UI or complete App transport acceptance. The product wire and diagnostic implementation remain unchanged. -These checks establish startup-order and ownership constraints, not a product -integration or a Piik field defect. The earlier candidate gate and unsupported +These transport checks establish startup-order and ownership constraints, not a +product integration or the cause of unmatched field reports. The earlier candidate gate and unsupported dependency method are not accepted product mechanisms. The existing assigned-connection and candidate owners must order admission and advertisement so the usable tunnel receives live ICE checks; ordinary ICE need diff --git a/docs/research/native-client-lifecycle.md b/docs/research/native-client-lifecycle.md index b6e2e30a5..31da5b30b 100644 --- a/docs/research/native-client-lifecycle.md +++ b/docs/research/native-client-lifecycle.md @@ -37,6 +37,19 @@ waiting; later grants can be used by later connections without replacing healthy Browser media. These checks do not identify the original missing-window reporter's cause or establish other browsers' permission behavior. +### Viewer Control Retirement + +A denied Viewer page can stay mounted or enter the Browser's +[back/forward cache](https://web.dev/articles/bfcache). Chromium 152 retained its +App control socket after invitation revocation; repeated entry eventually +exhausted the App's two control slots and used Browser reception instead. +The Viewer room owner now releases control on access loss and rejects discovery +that completes after retirement. A rendered optional-transport fixture retained +the denied pages across four invitation generations: each new generation used +Native reception with changing decoded pixels, old grants remained rejected, +and revocation cleared media resources. This is local synthetic-media evidence; +it does not establish physical network or all Browser navigation behavior. + ## Scope Decisions - Keep one strict loopback wire and one-share/one-session fence. diff --git a/docs/standards/presentation-lifecycle.md b/docs/standards/presentation-lifecycle.md index c511e28ab..ab386115d 100644 --- a/docs/standards/presentation-lifecycle.md +++ b/docs/standards/presentation-lifecycle.md @@ -28,7 +28,8 @@ A capable, opted-in authenticated room session outlives its current publication. Stopping capture retires media, not chat; restoring an existing room joins it without starting capture. Entering the page alone does not create a room. Room close, lost authority and page teardown still -retire the session. Connected Viewers may keep interacting while the Host page +retire the session, including its App control connection and pending discovery. +That retirement cannot wait for the denied page to unmount. Connected Viewers may keep interacting while the Host page is absent, provided the room service and their admitted sessions remain available. [Rooms and access](./rooms-access.md#room-interactions) owns interaction authority and retention; [status](../status.md#accepted-interaction-phase) owns release readiness. diff --git a/docs/todo.md b/docs/todo.md index 31d8d1523..144416620 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -80,8 +80,8 @@ Implementation needs a concrete useful result within the product model. Obtain supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary), direct-media, endpoint-discovery and RTCP/MTU contracts plus a trusted relay configuration. Experimental dependency patches are not product APIs. - Extend optional-path acceptance to invitation revocation and overlapping-parent - transitions; bounded rendered chain/reconnect/departure checks pass. Preserve + Extend optional-path acceptance to overlapping-parent transitions; bounded + rendered chain/reconnect/departure and invitation-revocation checks pass. Preserve ordinary ICE when bootstrap is unavailable. The Native composition still needs a private socket-reservation API and a multi-mux address-lookup correction; verify physical interface changes and direct-only policy before adoption. diff --git a/src/client/pages/ViewerPage.tsx b/src/client/pages/ViewerPage.tsx index 71db4c215..da1177c65 100644 --- a/src/client/pages/ViewerPage.tsx +++ b/src/client/pages/ViewerPage.tsx @@ -90,6 +90,7 @@ import { deriveViewerPresentation, reduceViewerPresentation, viewerFailureFromServerCode, + type ViewerFailureCode, type ViewerPresentationAction, type ViewerRouteKind, } from "../media/viewer-presentation"; @@ -483,11 +484,17 @@ export function ViewerPage({ let nativeViewerAvailable = true; const nativeViewerSessionId = createOpaqueId(); + const releaseNativeClient = (): void => { + const connecting = nativeClientPromise; + nativeClientPromise = null; + void connecting?.then((client) => client?.close()); + }; + const acquireNativeClient = async (): Promise => { if (!active || !launchedByClient || !nativeViewerAvailable) return null; if (!nativeClientPromise) { const connecting: Promise = NativeClient.connect({ waitForPermission: false }).catch(() => null).then((client) => { - if (!client || !active) { + if (!client || !active || nativeClientPromise !== connecting) { if (nativeClientPromise === connecting) nativeClientPromise = null; client?.close(); return null; @@ -581,29 +588,11 @@ export function ViewerPage({ if (!active) { return; } - messageAuthority.invalidate(); - setAssignedRoute(null); - clearViewerSfuRoute(); - clearPeerState(true); - clearParticipantPresence(); - dispatchPresentation({ - type: "access", - access: "denied", - failure: reason, - }); + denyRoomAccess(reason); }, onAccessRequired: () => { if (active) { - messageAuthority.invalidate(); - setAssignedRoute(null); - clearViewerSfuRoute(); - clearPeerState(true); - clearParticipantPresence(); - dispatchPresentation({ - type: "access", - access: "denied", - failure: "INVALID_TOKEN", - }); + denyRoomAccess("INVALID_TOKEN"); } }, onMessage: (message) => { @@ -1243,6 +1232,17 @@ export function ViewerPage({ viewerRelay?.stop(); } + function denyRoomAccess(failure: ViewerFailureCode): void { + messageAuthority.invalidate(); + setAssignedRoute(null); + clearViewerSfuRoute(); + clearPeerState(true); + clearParticipantPresence(); + // A denied page can remain mounted or enter the back/forward cache. + releaseNativeClient(); + dispatchPresentation({ type: "access", access: "denied", failure }); + } + function applyMediaAssignment(assignment: MediaAssignment, preserveUpstream = false): void { const nextAssignment = limitMediaAssignment( assignment, @@ -1776,43 +1776,19 @@ export function ViewerPage({ } viewerAuthorizationGeneration = null; clearViewerGrant(roomId); - setAssignedRoute(null); - clearViewerSfuRoute(); - clearPeerState(true); - clearParticipantPresence(); - dispatchPresentation({ - type: "access", - access: "denied", - failure: "INVALID_TOKEN", - }); + denyRoomAccess("INVALID_TOKEN"); signal.stop(); return; } if (message.type === "room-closed") { - setAssignedRoute(null); - clearViewerSfuRoute(); - clearPeerState(true); - clearParticipantPresence(); - dispatchPresentation({ - type: "access", - access: "denied", - failure: "ROOM_CLOSED", - }); + denyRoomAccess("ROOM_CLOSED"); signal.stop(); return; } if (message.type === "error") { const failure = viewerFailureFromServerCode(message.code); if (failure !== null && failure !== "SERVER_ERROR") { - setAssignedRoute(null); - clearViewerSfuRoute(); - clearPeerState(true); - clearParticipantPresence(); - dispatchPresentation({ - type: "access", - access: "denied", - failure, - }); + denyRoomAccess(failure); } if (message.code === "INVALID_TOKEN") { clearViewerGrant(roomId); @@ -1866,7 +1842,7 @@ export function ViewerPage({ viewerRelay?.dispose(); viewerRelay = null; viewerRelaySourceKey = null; - void nativeClientPromise?.then((client) => client?.close()); + releaseNativeClient(); }; }, [roomId, viewerGrant, viewerPasswordAttempt, launchedByClient, connectionAttemptProgress4]); diff --git a/tests/viewer-native-client-ownership.test.ts b/tests/viewer-native-client-ownership.test.ts index 7e799eb83..d16e5c7c2 100644 --- a/tests/viewer-native-client-ownership.test.ts +++ b/tests/viewer-native-client-ownership.test.ts @@ -8,26 +8,28 @@ import { NativeCompatibilityError } from "../src/client/native/client"; const source = ts.createSourceFile("ViewerPage.tsx", readFileSync( new URL("../src/client/pages/ViewerPage.tsx", import.meta.url), "utf8", ), ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX); -const owners = new Set(["nativeClientPromise", "nativeViewerAvailable", "acquireNativeClient"]); +const owners = new Set(["nativeClientPromise", "nativeViewerAvailable", "acquireNativeClient", "releaseNativeClient"]); const declarations: string[] = []; -let cleanup = ""; +let denyAccess = ""; function collect(node: ts.Node): void { if (ts.isVariableDeclaration(node) && owners.has(node.name.getText(source))) { declarations.push(`let ${node.getText(source)};`); } - if (ts.isExpressionStatement(node) && node.getText(source).startsWith("void nativeClientPromise?.then")) { - cleanup = node.getText(source); + if (ts.isFunctionDeclaration(node) && node.name?.text === "denyRoomAccess") { + denyAccess = node.getText(source); } ts.forEachChild(node, collect); } collect(source); -if (declarations.length !== owners.size || !cleanup) throw new Error("Viewer Native owner was not found"); +if (declarations.length !== owners.size || !denyAccess) throw new Error("Viewer Native owner was not found"); const executable = ts.transpile(` let active = true; ${declarations.join("\n")} + ${denyAccess} globalThis.acquire = acquireNativeClient; + globalThis.deny = denyRoomAccess; globalThis.disable = () => { nativeViewerAvailable = false; }; - globalThis.dispose = () => { active = false; ${cleanup} }; + globalThis.dispose = () => { active = false; releaseNativeClient(); }; `, { target: ts.ScriptTarget.ES2022 }); function control() { @@ -41,12 +43,18 @@ function control() { function fixture(launchedByClient = true) { const connect = vi.fn(async (): Promise | null> => null); - const context = createContext({ launchedByClient, NativeClient: { connect } }); + const context = createContext({ + launchedByClient, NativeClient: { connect }, + messageAuthority: { invalidate: vi.fn() }, setAssignedRoute: vi.fn(), + clearViewerSfuRoute: vi.fn(), clearPeerState: vi.fn(), clearParticipantPresence: vi.fn(), + dispatchPresentation: vi.fn(), + }); runInContext(executable, context); return { connect, acquire: context.acquire as () => Promise | null>, disable: context.disable as () => void, + deny: context.deny as (reason: string) => void, dispose: context.dispose as () => void, }; } @@ -67,6 +75,40 @@ it.each([false, true])("retries an absent or rejected App on the next offer: rej expect(client.close).toHaveBeenCalledOnce(); }); +it.each(["INVALID_TOKEN", "ROOM_CLOSED", "SESSION_REPLACED", "ROOM_ACCESS_DENIED"])( + "releases Native control on %s without waiting for the denied page to unmount", async (reason) => { + const current = fixture(); + const client = control(); + current.connect.mockResolvedValueOnce(client); + await current.acquire(); + current.deny(reason); + await Promise.resolve(); + expect(client.close).toHaveBeenCalledOnce(); + current.dispose(); + await Promise.resolve(); + expect(client.close).toHaveBeenCalledOnce(); + }, +); + +it("retires pending discovery after access loss without replacing or closing a newer connection", async () => { + const current = fixture(); + const obsolete = control(); + let resolve!: (value: typeof obsolete) => void; + current.connect.mockReturnValueOnce(new Promise((done) => { resolve = done; })); + const pending = current.acquire(); + current.deny("INVALID_TOKEN"); + const replacement = control(); + current.connect.mockResolvedValueOnce(replacement); + await expect(current.acquire()).resolves.toBe(replacement); + resolve(obsolete); + await expect(pending).resolves.toBeNull(); + expect(obsolete.close).toHaveBeenCalledOnce(); + expect(replacement.close).not.toHaveBeenCalled(); + await expect(current.acquire()).resolves.toBe(replacement); + expect(current.connect).toHaveBeenCalledTimes(2); + current.dispose(); +}); + it("keeps Browser fallback available after an App mismatch and retries on the next offer", async () => { const current = fixture(); current.connect.mockRejectedValueOnce(new NativeCompatibilityError(8)); diff --git a/tests/viewer-route-health.test.ts b/tests/viewer-route-health.test.ts index b41dd0b0e..6000fa40a 100644 --- a/tests/viewer-route-health.test.ts +++ b/tests/viewer-route-health.test.ts @@ -9,7 +9,7 @@ import type { ViewerPeerEvents } from "../src/client/webrtc/viewer-peer"; const source = ts.createSourceFile("ViewerPage.tsx", readFileSync( new URL("../src/client/pages/ViewerPage.tsx", import.meta.url), "utf8", ), ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX); -const owners = new Set(["ensurePeer", "reportActivePeerFailure"]); +const owners = new Set(["ensurePeer", "reportActivePeerFailure", "noteReceiveFailure"]); const functions: string[] = []; function collect(node: ts.Node): void { if (ts.isFunctionDeclaration(node) && node.name && owners.delete(node.name.text)) { From e405e5ace1380ee32feb946aea6514642616ed77 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:30:15 +0800 Subject: [PATCH 132/262] docs(network): record optional transport acceptance evidence --- docs/research/nat-traversal.md | 15 +++++++++++++-- docs/todo.md | 4 ++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index c358e1397..1bfe27f13 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1175,8 +1175,19 @@ acceptance. Optional invitation revocation also rejects old links, clears media resources and permits new Native reception across four grant generations. This check exposed and now covers the shared Viewer's [control-retirement gap](./native-client-lifecycle.md#viewer-control-retirement). -Optional overlapping-parent checks remain separate from the earlier tunnel-only -UI evidence. +A second optional UI check moves one Native Viewer between two Native parents +in both directions through the real room controller. Holding candidate readiness +within the unchanged operation deadline preserves the old video element, stream +and advancing frames while both receiver flows exist. Releasing readiness commits +the new parent and retires only the old flow; a sibling retains its connection +and advancing decoded pictures throughout. Final transport counts reach zero +under the race detector. The fixture disables automatic quality tuning and +changes advertised capacity to isolate overlapping ownership. This local synthetic +check does not measure handoff latency, difficult-network success or tunnel selection. +The optional composition separately passes the kernel-NAT unchanged-path control +with zero relayed encrypted data, retaining fresh Native playback packets after DERP +closes. This fixed-encoded-input check establishes direct transport through the +virtual socket, not rendered App recovery or another reachable NAT class. This composition also exposes an upstream address-lookup mismatch: Pion's multi-mux enumerates current child addresses but `GetConn` only consults its diff --git a/docs/todo.md b/docs/todo.md index 144416620..bfede8972 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -80,8 +80,8 @@ Implementation needs a concrete useful result within the product model. Obtain supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary), direct-media, endpoint-discovery and RTCP/MTU contracts plus a trusted relay configuration. Experimental dependency patches are not product APIs. - Extend optional-path acceptance to overlapping-parent transitions; bounded - rendered chain/reconnect/departure and invitation-revocation checks pass. Preserve + Bounded optional-path checks cover rendered chain/reconnect/departure, + invitation revocation and overlapping-parent transitions in both directions. Preserve ordinary ICE when bootstrap is unavailable. The Native composition still needs a private socket-reservation API and a multi-mux address-lookup correction; verify physical interface changes and direct-only policy before adoption. From 25630fe5eb7e07da1c072b3ea6717d3e839f1220 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 18:02:25 +0800 Subject: [PATCH 133/262] docs(network): record optional transport recovery and field-test boundary --- docs/research/nat-traversal.md | 21 +++++++++++++++++---- docs/todo.md | 3 ++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 1bfe27f13..1e89227b7 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1184,10 +1184,23 @@ and advancing decoded pictures throughout. Final transport counts reach zero under the race detector. The fixture disables automatic quality tuning and changes advertised capacity to isolate overlapping ownership. This local synthetic check does not measure handoff latency, difficult-network success or tunnel selection. -The optional composition separately passes the kernel-NAT unchanged-path control -with zero relayed encrypted data, retaining fresh Native playback packets after DERP -closes. This fixed-encoded-input check establishes direct transport through the -virtual socket, not rendered App recovery or another reachable NAT class. +The optional composition also passes the kernel-NAT real-codec controls with +Chromium 152 and Firefox 151: unchanged paths, either endpoint's address change, +and a short complete interruption. Ordinary Pion supplies unchanged/interruption +controls. New blue content renders on the same track after DERP closes, with +zero relayed encrypted data. Firefox renegotiates its local leg once when its +endpoint moves; Chromium retains that leg. No timeout or address policy changes. +These synthetic-source checks exercise the Native engine and Browser bridge, +not complete App startup, physical roaming or another reachable NAT class. + +A non-race Chromium comparison also measures the isolated Go workers. One +roughly 18-second, 640x360/30 connection used peak RSS of 64/65 MiB for ordinary +sender/receiver workers and 72/72 MiB with the optional tunnel. Live goroutines +rose from 39/65 to 96/122; final cumulative CPU times were 0.48/0.54 seconds and +0.67/0.67 seconds respectively. These are bounded observations, not App budgets: +the codec, Browser and relay run separately, and both controls link the same +binary. Shared listeners and clients also prevent linear extrapolation to a +room. Full App package size, sustained load and multi-peer cost remain unmeasured. This composition also exposes an upstream address-lookup mismatch: Pion's multi-mux enumerates current child addresses but `GetConn` only consults its diff --git a/docs/todo.md b/docs/todo.md index bfede8972..7ea8381c5 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -91,7 +91,8 @@ Implementation needs a concrete useful result within the product model. change or product transport integration is accepted; all experimental wire changes remain isolated. The P2P evidence work below owns network/interop acceptance. -- [ ] **Small external test before 2.0.** First use the isolated transport +- [ ] **Small external test before 2.0.** The owner will arrange testers after + the internal package and paired comparison steps are ready. First use the isolated transport experiment on the same difficult network pairs as current Piik, measuring direct success separately from relay delivery, first-picture time, recovery and added resource cost. This evidence can precede other feature work. From ff147f0535fcce720042a0c413580d55e6a18b03 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 18:34:14 +0800 Subject: [PATCH 134/262] docs(network): record complete App fallback evidence --- docs/research/nat-traversal.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 1e89227b7..a5625583b 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1200,7 +1200,20 @@ rose from 39/65 to 96/122; final cumulative CPU times were 0.48/0.54 seconds and 0.67/0.67 seconds respectively. These are bounded observations, not App budgets: the codec, Browser and relay run separately, and both controls link the same binary. Shared listeners and clients also prevent linear extrapolation to a -room. Full App package size, sustained load and multi-peer cost remain unmeasured. +room. A stripped Windows App build grows from 44.1 to 59.8 MiB with the isolated +composition, using the same embedded UI and module selection. This excludes +capture/tunnel helpers and ZIP compression. Sustained and multi-peer cost remain +unmeasured. + +The isolated App overlay now installs that constructor in the actual process-level +control factory. Its matched experimental wire identity rejects product builds, +local CLI configuration selects the TLS bootstrap, and DERP media is disabled in +the binary rather than by an optional environment setting. Actual Windows App +and embedded-UI checks retain fresh decoded pixels and manual reconnect through +ordinary ICE when the experimental bootstrap is unavailable. Two independent +control sessions and process retirement also pass a race check. Successful +bootstrap through the full App and a distributable test package remain open; +these controls do not establish field success or physical capture compatibility. This composition also exposes an upstream address-lookup mismatch: Pion's multi-mux enumerates current child addresses but `GetConn` only consults its From e9538f5d202ea01d1fcde7541ed77e9fbd6e7897 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:09:58 +0800 Subject: [PATCH 135/262] docs(preview): record internal package and field acceptance boundary --- docs/research/nat-traversal.md | 19 ++++++++++++++++--- docs/status.md | 12 +++++++----- docs/todo.md | 11 +++++------ 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index a5625583b..aec5032fb 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1211,9 +1211,22 @@ local CLI configuration selects the TLS bootstrap, and DERP media is disabled in the binary rather than by an optional environment setting. Actual Windows App and embedded-UI checks retain fresh decoded pixels and manual reconnect through ordinary ICE when the experimental bootstrap is unavailable. Two independent -control sessions and process retirement also pass a race check. Successful -bootstrap through the full App and a distributable test package remain open; -these controls do not establish field success or physical capture compatibility. +control sessions and process retirement also pass a race check. The Linux full-App +fixture also selects Magicsock with ordinary remote candidates withheld, renders +generated Browser pixels through both Native sessions, and restores pictures on +a new selected connection after manual Viewer reconnect. Closing its process-owned +TLS bootstrap still leaves 300 new frames over ten seconds at 640x360/30, with +changed decoded pixels; App retirement passes the race check. This is local +integration evidence, not another reachable NAT class. + +A frozen Windows internal package includes the numeric-media candidate, matched +experimental UI, helpers and notices, separate ordinary/experimental launchers +and bilingual paired-test instructions. Final-package checks cover native discovery, +two control sessions, unselected-origin denial, shutdown and Explorer ZIP members +with extracted-file hashes. Its experimental launcher selects the +[upstream public bootstrap](https://github.com/tailscale/tailcat#tailcat) for the +future paired test; local acceptance used owned or unavailable bootstrap only. +Physical capture, public-invitation operation and field benefit remain open. This composition also exposes an upstream address-lookup mismatch: Pion's multi-mux enumerates current child addresses but `GetConn` only consults its diff --git a/docs/status.md b/docs/status.md index ff0bf8a74..e7c9edee6 100644 --- a/docs/status.md +++ b/docs/status.md @@ -1,6 +1,6 @@ # Current Status -Last updated: 2026-10-09 +Last updated: 2026-10-10 This is the compact execution/deployment index. Product modules own behavior, [verification status](./verification-status.md) owns unresolved physical limits, @@ -72,10 +72,12 @@ The [media assessment](./research/native-client-media.md#media-capability-extens owns local codec, derived-output, SFU, Browser/Native fanout and live-control evidence. macOS geometry, cross-browser/device and route-change acceptance, and additional codecs remain open. -Native Magicsock remains an isolated experiment with incomplete product -integration and unresolved recovery cases. These proposals do not yet change -the accepted contract or transport. A small external test is proposed before -release; publication remains subject to the owner's final acceptance. +Native Magicsock remains an isolated experiment. Full-App local transport and +reconnect checks pass, and a frozen Windows internal package is available for +paired testing. Physical-network benefit, sustained recovery and supported +dependency contracts remain open; this does not accept product integration. +The owner will arrange external testers. Publication remains subject to final +acceptance. Immutable release artifacts own delivery completion; [TODO](./todo.md#now) owns remaining work. diff --git a/docs/todo.md b/docs/todo.md index 7ea8381c5..c26546aef 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -91,15 +91,14 @@ Implementation needs a concrete useful result within the product model. change or product transport integration is accepted; all experimental wire changes remain isolated. The P2P evidence work below owns network/interop acceptance. -- [ ] **Small external test before 2.0.** The owner will arrange testers after - the internal package and paired comparison steps are ready. First use the isolated transport +- [ ] **Small external test before 2.0.** The frozen Windows internal package and + paired comparison steps are ready; the owner will arrange testers. Use the isolated transport experiment on the same difficult network pairs as current Piik, measuring direct success separately from relay delivery, first-picture time, recovery and added resource cost. This evidence can precede other feature work. - A combined product test package requires room-scoped authorization/revocation, - mixed Browser/Native behavior and bounded recovery to pass first. Freeze and - identify each test build; do not describe the present Magicsock experiment as - a feature-complete product awaiting only acceptance. The final candidate then + Keep each test build frozen and identifiable. The combined local package and + bounded authorization/recovery evidence do not establish a feature-complete + product or additional real-network reachability. The final candidate then receives the combined feature/UI and regression check. - [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) identifies UDP/storage requirements but establishes no cost saving. Keep the From caec5810595f1426574ca4c89c4c3904f7528285 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:26:46 +0800 Subject: [PATCH 136/262] docs(preview): require a coherent candidate before external testing --- docs/research/nat-traversal.md | 16 ++++++++++++++-- docs/status.md | 6 +++--- docs/todo.md | 10 +++++++--- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index aec5032fb..bee317aa3 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1202,8 +1202,8 @@ the codec, Browser and relay run separately, and both controls link the same binary. Shared listeners and clients also prevent linear extrapolation to a room. A stripped Windows App build grows from 44.1 to 59.8 MiB with the isolated composition, using the same embedded UI and module selection. This excludes -capture/tunnel helpers and ZIP compression. Sustained and multi-peer cost remain -unmeasured. +capture/tunnel helpers and ZIP compression. Sustained and full-room cost remain +open. The isolated App overlay now installs that constructor in the actual process-level control factory. Its matched experimental wire identity rejects product builds, @@ -1219,6 +1219,18 @@ TLS bootstrap still leaves 300 new frames over ten seconds at 640x360/30, with changed decoded pixels; App retirement passes the race check. This is local integration evidence, not another reachable NAT class. +A non-race full-App comparison uses one Native and three plain-Browser Viewers +with a generated 640x360/30 source. Each Viewer receives 300–301 new frames and +changing pixels over its ten-second observation. Ordinary/optional runs peak +at 81.8/93.2 MiB RSS and use 3.76/4.08 seconds of Go CPU during the roughly +forty-second observation sequence. Live goroutines fall from 211/327 to 27 in +both modes after tabs close, then the App exits normally. The optional run +selects Magicsock on the Native pair with ordinary remote candidates withheld; +Web edges retain ordinary ICE. Both measurements include the same owned TLS +bootstrap/STUN fixture and exclude separate Browser/codec processes. This is a +bounded mixed-room cost and retirement check, not endurance, a 20-Viewer budget +or a network-success comparison. + A frozen Windows internal package includes the numeric-media candidate, matched experimental UI, helpers and notices, separate ordinary/experimental launchers and bilingual paired-test instructions. Final-package checks cover native discovery, diff --git a/docs/status.md b/docs/status.md index e7c9edee6..bb37ad6fe 100644 --- a/docs/status.md +++ b/docs/status.md @@ -74,10 +74,10 @@ evidence. macOS geometry, cross-browser/device and route-change acceptance, and additional codecs remain open. Native Magicsock remains an isolated experiment. Full-App local transport and reconnect checks pass, and a frozen Windows internal package is available for -paired testing. Physical-network benefit, sustained recovery and supported +engineering checks. The owner has not used it and will arrange paired testing +once the main features form a coherent usable candidate. Physical-network benefit, sustained recovery and supported dependency contracts remain open; this does not accept product integration. -The owner will arrange external testers. Publication remains subject to final -acceptance. +Publication remains subject to final acceptance. Immutable release artifacts own delivery completion; [TODO](./todo.md#now) owns remaining work. diff --git a/docs/todo.md b/docs/todo.md index c26546aef..66b26005b 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -92,10 +92,14 @@ Implementation needs a concrete useful result within the product model. changes remain isolated. The P2P evidence work below owns network/interop acceptance. - [ ] **Small external test before 2.0.** The frozen Windows internal package and - paired comparison steps are ready; the owner will arrange testers. Use the isolated transport - experiment on the same difficult network pairs as current Piik, measuring + paired comparison steps are engineering evidence only. The owner has not used + that package and will arrange testers after the main features, especially + Magicsock, form a coherent usable candidate. Finish local App/room lifecycle, + mixed Browser/Native behavior, recovery, resource and dependency boundaries + before requesting that test. Use the candidate on the same difficult network + pairs as current Piik, measuring direct success separately from relay delivery, first-picture time, recovery - and added resource cost. This evidence can precede other feature work. + and added resource cost. Keep each test build frozen and identifiable. The combined local package and bounded authorization/recovery evidence do not establish a feature-complete product or additional real-network reachability. The final candidate then From 78b26230e4cab2575f888c16bb02de295c3aaf19 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:35:14 +0800 Subject: [PATCH 137/262] docs(network): record bounded full-room App acceptance --- docs/research/nat-traversal.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index bee317aa3..2a72b91c3 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1202,8 +1202,7 @@ the codec, Browser and relay run separately, and both controls link the same binary. Shared listeners and clients also prevent linear extrapolation to a room. A stripped Windows App build grows from 44.1 to 59.8 MiB with the isolated composition, using the same embedded UI and module selection. This excludes -capture/tunnel helpers and ZIP compression. Sustained and full-room cost remain -open. +capture/tunnel helpers and ZIP compression. Sustained cost remains open. The isolated App overlay now installs that constructor in the actual process-level control factory. Its matched experimental wire identity rejects product builds, @@ -1231,6 +1230,13 @@ bootstrap/STUN fixture and exclude separate Browser/codec processes. This is a bounded mixed-room cost and retirement check, not endurance, a 20-Viewer budget or a network-success comparison. +A bounded full-room run with one Native and nineteen Web Viewers also retains +changing 640x360/30 pictures: each ten-second observation advances 296–301 frames. +The twenty-first code-only attempt receives the existing generic admission +denial, a retained Viewer keeps receiving, and the App exits normally after +all tabs close. The Go process peaks at 99.3 MiB RSS. These Viewers share one +machine; this does not cover twenty Native peers or physical-network endurance. + A frozen Windows internal package includes the numeric-media candidate, matched experimental UI, helpers and notices, separate ordinary/experimental launchers and bilingual paired-test instructions. Final-package checks cover native discovery, From 0a6582f151ec4693ab6a239bd4bc2c0ce11447e8 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 21:48:51 +0800 Subject: [PATCH 138/262] docs(network): prioritize browser-compatible connectivity --- docs/research/nat-traversal.md | 42 ++++++++++++++++++++++- docs/status.md | 10 +++--- docs/todo.md | 62 +++++++++++----------------------- 3 files changed, 65 insertions(+), 49 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 2a72b91c3..83a25f5c6 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1,6 +1,6 @@ # Browser And Native NAT Traversal -Last reviewed: 2026-10-09 +Last reviewed: 2026-10-10 This document owns evidence for improving direct ICE without adding a new relay or a custom transport. The current product contract remains standard @@ -344,8 +344,48 @@ OS network-change detection, decoded playback or real-router behavior. Its seven-pair direct matrix matched Pion within four-second windows. Go reuse is feasible; product integration and network benefit require the controls below. +### Browser-Compatible Follow-Up + +The product target remains App-to-Browser and Browser-to-Browser sharing without +requiring every Viewer to install software. Existing IPv6, bounded gateway +mapping and standard ICE can benefit those edges; a new transport must be judged +against that same endpoint pair and relay policy. + +- The [2026 DCUtR measurement](https://arxiv.org/abs/2604.12484) covers 4.4 million + attempts across more than 85,000 networks. Its roughly 70% success is conditional + on relay reservation and address discovery; 97.6% of successful punches occur + on the first attempt. These are libp2p results, not a Piik baseline. They justify + separating preparation failures from hole-punch failures and measuring the + marginal benefit of retries, not copying a retry count or replacing ICE. +- [QUIC hole punching and migration](https://arxiv.org/abs/2408.01791) reports + shorter connection/restoration latency than its TCP/re-punching controls. + It does not demonstrate a drop-in improvement for Browser WebRTC. Current + [WebTransport](https://www.w3.org/TR/webtransport/) targets Browser-to-server + transport; it is not an arbitrary UDP socket for Magicsock. +- [ICE renomination](https://datatracker.ietf.org/doc/draft-thatcher-tsvwg-renomination/) + proposes changing the selected candidate pair without a full restart. It + remains an Internet-Draft. Pinned Pion WebRTC v4.2.19 already exposes + `SetICERenomination`; Piik does not enable it. Its ICE v4.4.0 implementation + references an earlier draft and automatically probes and reselects candidate + pairs. Verify wire-version agreement, Browser interoperability, probe cost and + recovery benefit before enabling it. This targets path changes after connection, + not a demonstrated increase in initial NAT reachability. Keep selection in the + framework; do not add a Piik path-selection loop. +- [Direct Sockets](https://developer.chrome.com/docs/iwa/direct-sockets) is + restricted to explicitly installed Isolated Web Apps. It does not remove the + installation requirement for an ordinary invitation page. + ### Integration Cost And Recovery Controls +**Current decision: no-go for the 2.0 candidate.** This composition requires +Native App transport at both ends of the enhanced edge. An App Host with a plain +Browser Viewer still uses ordinary WebRTC; mixed relaying does not enhance that +Browser edge. The owner therefore stopped integration and field-package work. +Retain the isolated evidence, not a product dependency or a promised feature. +Independently verified ordinary ICE/media repairs remain useful without this +stack. Reopen only for a concrete Browser-compatible benefit or an explicitly +accepted App-only scope; no additional reachable NAT class has been established. + The isolated `spike/magicsock-assessment` branch retains the comparison probes. The nested modules are outside Piik's build. Local bootstrap, peer denial and synthetic SRTP media checks passed. Initial withdrawal checks observed stopped diff --git a/docs/status.md b/docs/status.md index bb37ad6fe..8e2fd55fe 100644 --- a/docs/status.md +++ b/docs/status.md @@ -72,11 +72,11 @@ The [media assessment](./research/native-client-media.md#media-capability-extens owns local codec, derived-output, SFU, Browser/Native fanout and live-control evidence. macOS geometry, cross-browser/device and route-change acceptance, and additional codecs remain open. -Native Magicsock remains an isolated experiment. Full-App local transport and -reconnect checks pass, and a frozen Windows internal package is available for -engineering checks. The owner has not used it and will arrange paired testing -once the main features form a coherent usable candidate. Physical-network benefit, sustained recovery and supported -dependency contracts remain open; this does not accept product integration. +Magicsock adoption is no-go for this candidate: its App-to-App augmentation does +not improve the core App-to-Browser use case. The +[assessment](./research/nat-traversal.md#integration-cost-and-recovery-controls) +owns the retained experiment and product decision. Its package is retired; +independent ordinary ICE and media repairs remain in the candidate. Publication remains subject to final acceptance. Immutable release artifacts own delivery completion; [TODO](./todo.md#now) owns remaining work. diff --git a/docs/todo.md b/docs/todo.md index 66b26005b..ba1de36da 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -23,6 +23,15 @@ Git/PRs own completed history. ## Held By Owner +- [ ] **Magicsock adoption: no-go for this candidate.** Its direct-path + augmentation requires App transport at both ends and does not improve the + core App-to-Browser edge. Stop product integration, combined packaging and + Magicsock field-test recruitment. Retain isolated evidence in Git; preserve + independently verified ordinary ICE/media repairs. The + [assessment](./research/nat-traversal.md#integration-cost-and-recovery-controls) + owns the rationale. Reopen only for a concrete Browser-compatible benefit + with acceptable complexity, or a newly accepted App-only product scope. + - [ ] **Browser-internal encoder work.** Paused by the owner on 2026-10-01. [Comparisons and Chromium traces](./research/browser-local-encoding-pool.md#sustained-h264-recovery) own the evidence for synchronous H264 initialization, native adaptation and @@ -66,44 +75,10 @@ Implementation needs a concrete useful result within the product model. mismatch before accepting the replacement contract. The published 1.x enums remain that major's baseline; the PR is not a compatible enum-only extension of v1.9.2. -- [ ] **Native Magicsock.** Go reuse is feasible, but the - [matched comparisons](./research/nat-traversal.md#integration-cost-and-recovery-controls) - have not established another reachable NAT class. Verify the measured local - address-handoff benefit through the complete App and physical networks before - adoption. Bounded rendered Browser/Native chains cover reconnect, departure and - overlapping Native-parent changes. Sustained and difficult-network recovery - remain open. Preserve the existing Browser - sender, room-event and assigned-connection owners. Order admission and candidate - delivery without exhausting inner checks; diagnostic replies are not media-readiness evidence, and inner - checks may exhaust before the underlay recovers. Research owns the verified - sender/receiver command, key/lifecycle and local RTP evidence and their limits. - Obtain supported [authorization/revocation](./research/nat-traversal.md#peer-authorization-boundary), - direct-media, endpoint-discovery and RTCP/MTU contracts plus a trusted relay - configuration. Experimental dependency patches are not product APIs. - Bounded optional-path checks cover rendered chain/reconnect/departure, - invitation revocation and overlapping-parent transitions in both directions. Preserve - ordinary ICE when bootstrap is unavailable. The Native composition still needs - a private socket-reservation API and a multi-mux address-lookup correction; - verify physical interface changes and direct-only policy before adoption. - Preserve the local address policy (forcing loopback fails Firefox), existing - recovery deadlines and single route/resource owners. DERP must not become - another media fallback or bypass site/share policy. No new setting, timeout - change or product transport integration is accepted; all experimental wire - changes remain isolated. The P2P evidence work below owns network/interop - acceptance. -- [ ] **Small external test before 2.0.** The frozen Windows internal package and - paired comparison steps are engineering evidence only. The owner has not used - that package and will arrange testers after the main features, especially - Magicsock, form a coherent usable candidate. Finish local App/room lifecycle, - mixed Browser/Native behavior, recovery, resource and dependency boundaries - before requesting that test. Use the candidate on the same difficult network - pairs as current Piik, measuring - direct success separately from relay delivery, first-picture time, recovery - and added resource cost. - Keep each test build frozen and identifiable. The combined local package and - bounded authorization/recovery evidence do not establish a feature-complete - product or additional real-network reachability. The final candidate then - receives the combined feature/UI and regression check. +- [ ] **2.0 candidate acceptance.** Finish the combined numeric-media UI, + App/room lifecycle, ordinary Browser/Native routes and regression checks before + asking the owner to arrange external testing. Keep each test build frozen and + identifiable; the retired Magicsock package is not the current candidate. - [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) identifies UDP/storage requirements but establishes no cost saving. Keep the existing P2P-only Demo until a provider and operating boundary are accepted. @@ -234,11 +209,12 @@ to connection generations without raw endpoints. Verify survey responses and P2P/SFU handoffs, including background direct attempts behind working SFU media. The [iroh/Tailscale comparison](./research/nat-traversal.md#iroh-and-tailscale-comparison) records the isolated Go media/migration experiment and Browser/relay limits. -Before adopting magicsock in the product, establish a benefit on matched network -pairs and define room-authorized peer keys, endpoint exchange, MTU/resource limits -and mixed Browser/Native behavior. Keep the experiment isolated from the -production [routing contract](./standards/routing-transport.md); Go reuse already -works, so a Rust alternative needs a concrete Go limitation. +Prioritize one-App-to-Browser and Browser-to-Browser controls using existing +ICE, IPv6 and gateway mapping, with changing interfaces, lost mappings and +asymmetric network restrictions. The +[Browser-compatible assessment](./research/nat-traversal.md#browser-compatible-follow-up) +owns current literature and applicability. Do not infer a benefit from another +stack's success percentage or add a new retry/transport owner without evidence. ## Parked Product Work From 3148c19dea0f4a0c1b6e8c25743114a8211bb286 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:08:29 +0800 Subject: [PATCH 139/262] docs(network): record browser renomination boundary --- docs/research/nat-traversal.md | 21 +++++++++++++++++++++ docs/todo.md | 3 +++ 2 files changed, 24 insertions(+) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 83a25f5c6..a164a4a95 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -375,6 +375,27 @@ against that same endpoint pair and relay policy. restricted to explicitly installed Isolated Web Apps. It does not remove the installation requirement for an ordinary invitation page. +#### Pion Renomination Preflight + +The current decision is not to enable automatic renomination. An isolated +two-UDP-path control with pinned WebRTC v4.2.19/ICE v4.4.0 cut the selected socket +after stable VP8 playback. Chromium 152 recovered video and reverse feedback +without new SDP after about 5.6 seconds. Firefox 151.0.3 repeatedly resumed video +after about 5.7 seconds but retained the failed outbound candidate: RTCP reception +and echoed data stopped for the full 20-second observation while new video kept +decoding. An explicit ICE restart on that same connection selected the surviving +path and restored both reverse flows. The stock-ICE controls also recovered +through restart. + +Both Browsers answered with only `trickle`; the enabled Pion call nevertheless +sent the older nomination attribute. Thus receiving new frames alone cannot +establish compatible path migration. In these fixtures, steady Binding requests +also rose from 3 to 27-75 per five seconds, depending on interface enumeration. +These are local transport controls, not physical-network success measurements or +a defect in Piik's currently disabled feature. Reopen only for negotiated Browser +support and bounded probe/recovery behavior; do not add UA exceptions, a second +recovery loop or a Browser fork to enable this option. + ### Integration Cost And Recovery Controls **Current decision: no-go for the 2.0 candidate.** This composition requires diff --git a/docs/todo.md b/docs/todo.md index ba1de36da..e843c1e71 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -215,6 +215,9 @@ asymmetric network restrictions. The [Browser-compatible assessment](./research/nat-traversal.md#browser-compatible-follow-up) owns current literature and applicability. Do not infer a benefit from another stack's success percentage or add a new retry/transport owner without evidence. +The [renomination preflight](./research/nat-traversal.md#pion-renomination-preflight) +rejects enabling the current automatic option: Firefox's recovered video did not +restore reverse feedback. Keep standard ICE recovery while that boundary remains. ## Parked Product Work From 950de1d00ce6c12afe02e10819aa01288fcd0087 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:19:07 +0800 Subject: [PATCH 140/262] docs(network): assess browser-compatible upstream improvements --- docs/research/nat-traversal.md | 26 +++++++++++++++++++++++--- docs/todo.md | 3 +++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index a164a4a95..d10f40a7f 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -351,12 +351,17 @@ requiring every Viewer to install software. Existing IPv6, bounded gateway mapping and standard ICE can benefit those edges; a new transport must be judged against that same endpoint pair and relay policy. -- The [2026 DCUtR measurement](https://arxiv.org/abs/2604.12484) covers 4.4 million - attempts across more than 85,000 networks. Its roughly 70% success is conditional +- The [2026 DCUtR paper](https://arxiv.org/abs/2604.12484) covers 4.4 million + attempts across more than 85,000 networks, collected in December 2022 through + January 2023. Its roughly 70% success is conditional on relay reservation and address discovery; 97.6% of successful punches occur on the first attempt. These are libp2p results, not a Piik baseline. They justify separating preparation failures from hole-punch failures and measuring the - marginal benefit of retries, not copying a retry count or replacing ICE. + marginal benefit of retries, not copying a retry count or replacing ICE. Its + proposed role reversal and probabilistic port probing are future work, not + measured improvements in that campaign. Standard full ICE already checks in + both directions; a QUIC initiator-role proposal is not evidence for changing + Piik's Host authority or adding another retry owner. - [QUIC hole punching and migration](https://arxiv.org/abs/2408.01791) reports shorter connection/restoration latency than its TCP/re-punching controls. It does not demonstrate a drop-in improvement for Browser WebRTC. Current @@ -374,6 +379,21 @@ against that same endpoint pair and relay policy. - [Direct Sockets](https://developer.chrome.com/docs/iwa/direct-sockets) is restricted to explicitly installed Isolated Web Apps. It does not remove the installation requirement for an ordinary invitation page. +- [SPED](https://datatracker.ietf.org/doc/draft-hancke-webrtc-sped/) carries DTLS + handshake data in STUN so ICE and encryption setup can overlap. Its target is + setup latency and handshake loss recovery, not newly reachable NAT classes. + The March 2026 individual draft is expired as of this review; implementation + interest does not establish interoperable product support. Evaluate only + framework-owned negotiation and stock Browser builds, retaining authentication + and ordinary fallback. A Browser field-trial flag is not an adoption path. + +The 2026-10-10 upstream review found stable Pion WebRTC v4.2.23 / ICE v4.4.7 +beyond the current pins. The [ICE changes](https://github.com/pion/ice/compare/v4.4.0...v4.4.7) +include STUN retransmission, gathering-generation ownership, Chromium mDNS +compatibility and pre-created mux lifetime corrections. Inspect applicability +and test the shared Native/SFU composition before adopting them: this range also +changes STUN/transport package majors. A release-note match alone does not +establish a Piik defect or a connectivity gain. #### Pion Renomination Preflight diff --git a/docs/todo.md b/docs/todo.md index e843c1e71..44e83b061 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -215,6 +215,9 @@ asymmetric network restrictions. The [Browser-compatible assessment](./research/nat-traversal.md#browser-compatible-follow-up) owns current literature and applicability. Do not infer a benefit from another stack's success percentage or add a new retry/transport owner without evidence. +Review the newer stable Pion ICE/WebRTC fixes against the pinned Native/SFU +composition, especially gathering generations, STUN retransmission and mDNS; +accept dependency changes only after focused interoperability/recovery controls. The [renomination preflight](./research/nat-traversal.md#pion-renomination-preflight) rejects enabling the current automatic option: Firefox's recovered video did not restore reverse feedback. Keep standard ICE recovery while that boundary remains. From 04fd696e6d6a489b479f4074466c5a0ab681036b Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:47:41 +0800 Subject: [PATCH 141/262] fix(media): update Pion for browser ICE interoperability --- docs/research/nat-traversal.md | 43 +++++++++++---- docs/todo.md | 7 +-- go.mod | 27 +++++----- go.sum | 52 +++++++++---------- internal/app/mediaedge/ice_socket.go | 2 +- internal/app/mediaedge/receiver_test.go | 2 +- .../mediaedge/relay_network_fixture_test.go | 2 +- internal/app/mediaedge/stun.go | 2 +- internal/app/mediaedge/stun_test.go | 2 +- internal/media/forwarding/ice.go | 4 +- internal/media/forwarding/packet_buffer.go | 26 ++++++++++ .../media/forwarding/packet_buffer_test.go | 43 +++++++++++++++ internal/media/forwarding/publication.go | 4 +- internal/media/forwarding/recovery_test.go | 2 +- internal/media/forwarding/transport.go | 4 +- internal/server/stun/server_test.go | 2 +- 16 files changed, 160 insertions(+), 64 deletions(-) create mode 100644 internal/media/forwarding/packet_buffer.go create mode 100644 internal/media/forwarding/packet_buffer_test.go diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index d10f40a7f..e6f7f266e 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -369,8 +369,8 @@ against that same endpoint pair and relay policy. transport; it is not an arbitrary UDP socket for Magicsock. - [ICE renomination](https://datatracker.ietf.org/doc/draft-thatcher-tsvwg-renomination/) proposes changing the selected candidate pair without a full restart. It - remains an Internet-Draft. Pinned Pion WebRTC v4.2.19 already exposes - `SetICERenomination`; Piik does not enable it. Its ICE v4.4.0 implementation + remains an Internet-Draft. The tested Pion WebRTC v4.2.19 already exposed + `SetICERenomination`; Piik does not enable it. That ICE v4.4.0 implementation references an earlier draft and automatically probes and reselects candidate pairs. Verify wire-version agreement, Browser interoperability, probe cost and recovery benefit before enabling it. This targets path changes after connection, @@ -387,13 +387,38 @@ against that same endpoint pair and relay policy. framework-owned negotiation and stock Browser builds, retaining authentication and ordinary fallback. A Browser field-trial flag is not an adoption path. -The 2026-10-10 upstream review found stable Pion WebRTC v4.2.23 / ICE v4.4.7 -beyond the current pins. The [ICE changes](https://github.com/pion/ice/compare/v4.4.0...v4.4.7) -include STUN retransmission, gathering-generation ownership, Chromium mDNS -compatibility and pre-created mux lifetime corrections. Inspect applicability -and test the shared Native/SFU composition before adopting them: this range also -changes STUN/transport package majors. A release-note match alone does not -establish a Piik defect or a connectivity gain. +#### Stable Pion Interoperability + +The candidate uses Pion WebRTC v4.2.23 / ICE v4.4.7. The +[upstream changes](https://github.com/pion/ice/compare/v4.4.0...v4.4.7) include +STUN retransmission, gathering-generation ownership, Chromium mDNS compatibility +and pre-created mux lifetime corrections. This updates the shared Native/SFU +composition; ordinary Browser-to-Browser ICE remains Browser-owned. + +A controlled Chrome 152 check reproduced the +[mDNS query incompatibility](https://github.com/pion/mdns/commit/5b0a717e4ff6e81300d54ce2fef53e635b95fe3c): +after the Browser's initial announcement, a fresh v2.1.0 resolver could not resolve +its generated `.local` candidate before the fixture's four-second deadline. +v2.2.2 resolved the same name in about 12 ms; another fresh old resolver still +timed out. The upstream repair sends separate A/AAAA questions without the +unicast-response bit that Chromium rejects. This fixes an address-resolution +boundary useful to App-to-Browser edges; it is not a field success-rate result +or proof of the cause of an unmatched report. Peer-reflexive discovery can also +let a connection succeed despite failed mDNS resolution. + +The package-major adaptations retain Pion's STUN transaction owner and SRTP +buffer bounds/attributes. LiveKit forwarding and the existing ICE recovery +owners remain unchanged. Chrome 152 and Firefox 151.0.3 recovered synthetic VP8, +reverse RTCP and echoed data through ordinary ICE restart after the selected +UDP path was cut. Shared forwarding, Native edges, SFU and STUN tests passed, +including Linux race checks and a disposable-namespace interface-replacement +control. No new automatic renomination, probing loop or retry budget is enabled. + +Keep Piik's fresh-interface mux and connection-owned STUN/mapping repairs: the +newer framework still snapshots its default interface view and caches successful +universal-mux observations. An upstream release-note match is not sufficient to +remove these protections. Physical network/device coverage remains in +[verification status](../verification-status.md#remaining-device-and-network-acceptance). #### Pion Renomination Preflight diff --git a/docs/todo.md b/docs/todo.md index 44e83b061..cfb17f607 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -215,9 +215,10 @@ asymmetric network restrictions. The [Browser-compatible assessment](./research/nat-traversal.md#browser-compatible-follow-up) owns current literature and applicability. Do not infer a benefit from another stack's success percentage or add a new retry/transport owner without evidence. -Review the newer stable Pion ICE/WebRTC fixes against the pinned Native/SFU -composition, especially gathering generations, STUN retransmission and mDNS; -accept dependency changes only after focused interoperability/recovery controls. +The candidate's stable Pion upgrade has local Browser resolution and +interoperability/recovery evidence in that assessment. Include the upgraded +Native/SFU composition in physical-network acceptance; local success is not a +measured increase in field hole-punching success. The [renomination preflight](./research/nat-traversal.md#pion-renomination-preflight) rejects enabling the current automatic option: Firefox's recovered video did not restore reverse feedback. Keep standard ICE recovery while that boundary remains. diff --git a/go.mod b/go.mod index eaef584e7..8d97a75d7 100644 --- a/go.mod +++ b/go.mod @@ -21,16 +21,16 @@ require ( github.com/livekit/protocol v1.50.5-0.20260821144838-17c16cf496fd github.com/livekit/server-sdk-go/v2 v2.14.0 github.com/netbirdio/go-nat v0.0.0-20260821095157-6b2c8c5c74e8 - github.com/pion/ice/v4 v4.4.0 - github.com/pion/interceptor v0.1.47 + github.com/pion/ice/v4 v4.4.7 + github.com/pion/interceptor v0.1.49 github.com/pion/logging v0.2.4 - github.com/pion/rtcp v1.2.17 + github.com/pion/rtcp v1.2.19 github.com/pion/rtp v1.10.5 - github.com/pion/sdp/v3 v3.0.19 - github.com/pion/stun/v3 v3.1.7 - github.com/pion/transport/v4 v4.1.1 - github.com/pion/turn/v5 v5.0.13 - github.com/pion/webrtc/v4 v4.2.19 + github.com/pion/sdp/v3 v3.0.20 + github.com/pion/stun/v4 v4.0.1 + github.com/pion/transport/v5 v5.1.1 + github.com/pion/turn/v5 v5.1.2 + github.com/pion/webrtc/v4 v4.2.23 github.com/thesyncim/gopus v0.1.1 go.uber.org/zap v1.28.0 golang.org/x/crypto v0.56.0 @@ -96,12 +96,13 @@ require ( github.com/nats-io/nuid v1.0.1 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/petermattis/goid v0.0.0-20260725062400-500c67a39b75 // indirect - github.com/pion/datachannel v1.6.2 // indirect - github.com/pion/dtls/v3 v3.1.5 // indirect - github.com/pion/mdns/v2 v2.1.0 // indirect + github.com/pion/datachannel v1.6.3 // indirect + github.com/pion/dtls/v3 v3.1.10 // indirect + github.com/pion/mdns/v2 v2.2.2 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/sctp v1.11.1 // indirect - github.com/pion/srtp/v3 v3.0.13 // indirect + github.com/pion/sctp v1.12.0 // indirect + github.com/pion/srtp/v3 v3.1.3 // indirect + github.com/pion/transport/v4 v4.1.1 // indirect github.com/prometheus/client_golang v1.24.1 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect diff --git a/go.sum b/go.sum index d5c5957be..00b20a39a 100644 --- a/go.sum +++ b/go.sum @@ -235,40 +235,40 @@ github.com/ory/dockertest/v4 v4.0.0 h1:i19aFsO/VXE0VrMk4ifnKW4G/KIJ93PCjLOslxXoP github.com/ory/dockertest/v4 v4.0.0/go.mod h1:b5Ofu8VIxWNhXFvQcLu17pRNQdoUBKtXBW74G4Ygzx8= github.com/petermattis/goid v0.0.0-20260725062400-500c67a39b75 h1:VmZ6mKVkxavKEhEy4ZYyV7BwBYBFBP0TwIqmLk84fpU= github.com/petermattis/goid v0.0.0-20260725062400-500c67a39b75/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= -github.com/pion/datachannel v1.6.2 h1:7EXQ8TH3vTouBUdRWYbcX2edSx9Yj6k5zl5P+qyxEPc= -github.com/pion/datachannel v1.6.2/go.mod h1:pzbdAZvyGtXbcHM1hBbsFaOTf40lZizU/dNlvVOak6E= -github.com/pion/dtls/v3 v3.1.5 h1:9xJtVsHwMYeSjPp5Hh1FTis4DchnQWtnOa5o+6ygqfc= -github.com/pion/dtls/v3 v3.1.5/go.mod h1:gz1K4jg6c+fq86oQMH4pilpCEOEPwmEr2jY+VcF/mkU= -github.com/pion/ice/v4 v4.4.0 h1:wvHDDqimaC38Y7MVpD46Y63p246ChvXd87VKoLYS5b4= -github.com/pion/ice/v4 v4.4.0/go.mod h1:obAyD+J+Hzs7QA7Y8YXHp5uIn6gb7z87pKedXZkrcFU= -github.com/pion/interceptor v0.1.47 h1:yw8t5pJ2f8t78NgU+8EmxhaqYLXS7uFCC/tAGOaSDBo= -github.com/pion/interceptor v0.1.47/go.mod h1:7yoRBzaIDETPC6cIN8Zj9EyGqHv1ImOpcTFPha6MuOM= +github.com/pion/datachannel v1.6.3 h1:HZAYLpunVI0xi057ZiANl558d0uSpomcWw4UEPz3qXs= +github.com/pion/datachannel v1.6.3/go.mod h1:oRmkXVVXxkUbqa8MatgdNHW29kvtQY3HUdjJQsfv2GE= +github.com/pion/dtls/v3 v3.1.10 h1:HWC+QCZitP/ApADS/6+g7UIw2YmLgoK3CsynnjPJgMo= +github.com/pion/dtls/v3 v3.1.10/go.mod h1:iKFQNYrjsN2TiA2YKKMqB9MOZaFpjFULBI/A4sW0eyc= +github.com/pion/ice/v4 v4.4.7 h1:JiI6VId3XnIztxKE8RjTC0SeXrp7ppkAO0jGez3/sIE= +github.com/pion/ice/v4 v4.4.7/go.mod h1:zCRRg4Mkpdbx9B+75fd4z2APvIXeIraeUOB7IMPt8mU= +github.com/pion/interceptor v0.1.49 h1:iyBsNHRoLNNXZiJA/DdGvJgdyfS8YWDCzxIAlVjt5nY= +github.com/pion/interceptor v0.1.49/go.mod h1:MZ6PJkja/TCo350HAnBrs/rUIyad9mWjpcvytrf3ViQ= github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= -github.com/pion/mdns/v2 v2.1.0 h1:3IJ9+Xio6tWYjhN6WwuY142P/1jA0D5ERaIqawg/fOY= -github.com/pion/mdns/v2 v2.1.0/go.mod h1:pcez23GdynwcfRU1977qKU0mDxSeucttSHbCSfFOd9A= +github.com/pion/mdns/v2 v2.2.2 h1:qq9Kj3PnjF0mfSwd8Icu0CXy5aoYdQKnNUIsGPeUpgk= +github.com/pion/mdns/v2 v2.2.2/go.mod h1:ZX5f0AAH1D6TOjdjvcBORZZHaZuG0t9+br78lHEwiJ4= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= -github.com/pion/rtcp v1.2.17 h1:PxiT6L79yPZKtXIsXdG1eakBl6dtBj4x+4oVEL0DlSw= -github.com/pion/rtcp v1.2.17/go.mod h1:7kBpuBJaWwax4hzc/pgexY8vkOpvh8atgYDbaKZq0iU= +github.com/pion/rtcp v1.2.19 h1:T79lR2Tfd/7zj+41nsUrftrUEee+Z03NQr9GSPSQ/WU= +github.com/pion/rtcp v1.2.19/go.mod h1:vX7Es6skWYm4oVvOSC5+4WHLYUTZHcbGNmk5wjzMGH4= github.com/pion/rtp v1.10.5 h1:ip0HhO/wYZqQ4bKS+R99KnZh/GRCmIT0jDXikub7vlE= github.com/pion/rtp v1.10.5/go.mod h1:Au8fc6cEByy8RLTwKTQTEeQqDB/SJDxwL4mZuxYA5Pk= -github.com/pion/sctp v1.11.1 h1:O4dIFyURw1KTST7w+gtD4gLeYXkhPa0xXLHMMoe/OSA= -github.com/pion/sctp v1.11.1/go.mod h1:7KFmTwLcoYgJs/Z+99nJvsWL0qDpuyloSI0RbAqlrz0= -github.com/pion/sdp/v3 v3.0.19 h1:1VMKs3gIkTQV5M3hNKfTAPrDXSNrYtOlmOD8+mSZUGQ= -github.com/pion/sdp/v3 v3.0.19/go.mod h1:dE5WOSlzXrtiE/iuZqe9n+AcEbOjtAd3k5m5NtlV/qU= -github.com/pion/srtp/v3 v3.0.13 h1:FmQaqgNbN1vUtMhEsmj8trldc3lNZr1xmN7nl8CyX+Q= -github.com/pion/srtp/v3 v3.0.13/go.mod h1:7qR3L69t8RX0EPVQwGNwCa1Gy9keKKNDpWwQzZbeXDY= -github.com/pion/stun/v3 v3.1.7 h1:uRXMTlGLf89WgItGNyZ6aR5jMTX0NBbybXADpQCzn+E= -github.com/pion/stun/v3 v3.1.7/go.mod h1:Nq77RW4aRrSNrltf2ksUJLjxWeipj4lnlgdsYIxC8g8= -github.com/pion/transport/v3 v3.1.1 h1:Tr684+fnnKlhPceU+ICdrw6KKkTms+5qHMgw6bIkYOM= -github.com/pion/transport/v3 v3.1.1/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ= +github.com/pion/sctp v1.12.0 h1:AzUdVkVBNyGY1W4crWO1/aOWzMaW3zsVHGrNuAfFQiI= +github.com/pion/sctp v1.12.0/go.mod h1:6k1zrpkrxSwbisitn/AlzDQaa15DR49MFsrMochG/yc= +github.com/pion/sdp/v3 v3.0.20 h1:TS6DViqcmp+49f0+mjw9anbr9xY3vJtsZewxAvlMCRQ= +github.com/pion/sdp/v3 v3.0.20/go.mod h1:slIMXDK5OKj0nhISwjfeN18AzTBCt2LYZq9uPw0cU5Q= +github.com/pion/srtp/v3 v3.1.3 h1:O3sfbfWfq/OyEaXRbX02Z0SNQRijS9nDdc/DEb8jxlU= +github.com/pion/srtp/v3 v3.1.3/go.mod h1:v3uQtrCz9LH3aahau0jtzRDsWGwKkh4a0BiBwzzxhtQ= +github.com/pion/stun/v4 v4.0.1 h1:S2ggK4hsUJJKoZfpL90uIqIRTj6xJ0NOIEkXgcpQif0= +github.com/pion/stun/v4 v4.0.1/go.mod h1:byktbPA8U7HjJ2H8w3moP7nzq0Q+kYNulvMLPaubQL4= github.com/pion/transport/v4 v4.1.1 h1:kEBtbp1sRad0tnkREDNJ3fmop/CR8p/YxXnpFbn9tqo= github.com/pion/transport/v4 v4.1.1/go.mod h1:Oxo5oODQvjFF7NkHBBz3f4fZXLB7x03uLVuzfYeA5pE= -github.com/pion/turn/v5 v5.0.13 h1:erHOsJyxuV6QK54+PjWJhe8u1O7BM3a/US0zYJJsnx4= -github.com/pion/turn/v5 v5.0.13/go.mod h1:btdOovUYdYc8iBnvt87JHN4Pa1XV5UiLaCYe4ay3o9A= -github.com/pion/webrtc/v4 v4.2.19 h1:2usG6s7eXMF08tqqoP3A4CX5XHArZsi1qeXDIIvXMeE= -github.com/pion/webrtc/v4 v4.2.19/go.mod h1:jWMn3KOGxyYFYJohzIHhBc4VrJiyS+94A1OfErYzN34= +github.com/pion/transport/v5 v5.1.1 h1:Sauq8YJqRopgRfIgnpfKpdpKG14g0N4jWlRgpWsEfN8= +github.com/pion/transport/v5 v5.1.1/go.mod h1:Qxw6fCEjFWQkRDZOhS4Vf+neJBcihauvA3uyEa1J1F0= +github.com/pion/turn/v5 v5.1.2 h1:acEOO+D5txJ52vKfcilmSMItfP23Q9K6pJ1LM5P2VV0= +github.com/pion/turn/v5 v5.1.2/go.mod h1:57Oadc8fHnac15VFyCd6nEaBTVZBxQe2POUMD4lsHRA= +github.com/pion/webrtc/v4 v4.2.23 h1:AlVixhTtQmxrheSv6lAH0w2sH5qjM7OTos5Lt811nk0= +github.com/pion/webrtc/v4 v4.2.23/go.mod h1:vzf5RQ0uMBuXAYTK2pabZUtnZyj2t3TF88eL4xzQZvI= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= diff --git a/internal/app/mediaedge/ice_socket.go b/internal/app/mediaedge/ice_socket.go index 0e458aee9..5c5ff9798 100644 --- a/internal/app/mediaedge/ice_socket.go +++ b/internal/app/mediaedge/ice_socket.go @@ -14,7 +14,7 @@ import ( "github.com/TNTcraftHIM/Piik/internal/diagnostics" "github.com/TNTcraftHIM/Piik/internal/media/forwarding" "github.com/pion/ice/v4" - "github.com/pion/stun/v3" + "github.com/pion/stun/v4" "github.com/pion/webrtc/v4" "golang.org/x/sync/singleflight" ) diff --git a/internal/app/mediaedge/receiver_test.go b/internal/app/mediaedge/receiver_test.go index af8ecc8dd..46e3c0a5d 100644 --- a/internal/app/mediaedge/receiver_test.go +++ b/internal/app/mediaedge/receiver_test.go @@ -12,7 +12,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu" "github.com/pion/rtp" "github.com/pion/sdp/v3" - "github.com/pion/stun/v3" + "github.com/pion/stun/v4" "github.com/pion/webrtc/v4" ) diff --git a/internal/app/mediaedge/relay_network_fixture_test.go b/internal/app/mediaedge/relay_network_fixture_test.go index 8f1958b0e..9d80ef25e 100644 --- a/internal/app/mediaedge/relay_network_fixture_test.go +++ b/internal/app/mediaedge/relay_network_fixture_test.go @@ -19,7 +19,7 @@ import ( "github.com/pion/rtcp" "github.com/pion/rtp" "github.com/pion/rtp/codecs" - "github.com/pion/transport/v4/vnet" + "github.com/pion/transport/v5/vnet" "github.com/pion/webrtc/v4" ) diff --git a/internal/app/mediaedge/stun.go b/internal/app/mediaedge/stun.go index c6e3b4e22..82dacd97e 100644 --- a/internal/app/mediaedge/stun.go +++ b/internal/app/mediaedge/stun.go @@ -6,7 +6,7 @@ import ( "net" "github.com/TNTcraftHIM/Piik/internal/diagnostics" - "github.com/pion/stun/v3" + "github.com/pion/stun/v4" ) // Only in-flight discovery is shared. Completed observations must not survive diff --git a/internal/app/mediaedge/stun_test.go b/internal/app/mediaedge/stun_test.go index d41f278ff..f4beaa41a 100644 --- a/internal/app/mediaedge/stun_test.go +++ b/internal/app/mediaedge/stun_test.go @@ -11,7 +11,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu" "github.com/pion/rtp" - "github.com/pion/stun/v3" + "github.com/pion/stun/v4" "github.com/pion/webrtc/v4" ) diff --git a/internal/media/forwarding/ice.go b/internal/media/forwarding/ice.go index cafec6f81..85c99f018 100644 --- a/internal/media/forwarding/ice.go +++ b/internal/media/forwarding/ice.go @@ -2,8 +2,8 @@ package forwarding import ( "github.com/pion/ice/v4" - "github.com/pion/transport/v4" - "github.com/pion/transport/v4/stdnet" + "github.com/pion/transport/v5" + "github.com/pion/transport/v5/stdnet" ) // NewUDPMux retains Pion's socket and candidate policy, but reads current local diff --git a/internal/media/forwarding/packet_buffer.go b/internal/media/forwarding/packet_buffer.go new file mode 100644 index 000000000..ebd6d514b --- /dev/null +++ b/internal/media/forwarding/packet_buffer.go @@ -0,0 +1,26 @@ +package forwarding + +import "github.com/pion/transport/v5/packetio" + +// packetBuffer adapts Pion's bounded packet queue to its SRTP BufferFactory's +// io.ReadWriteCloser contract, retaining optional packet attributes. +type packetBuffer struct { + *packetio.Buffer +} + +func (buffer *packetBuffer) Read(payload []byte) (int, error) { + n, _, err := buffer.Buffer.Read(payload, nil) + return n, err +} + +func (buffer *packetBuffer) Write(payload []byte) (int, error) { + return buffer.Buffer.Write(payload, nil) +} + +func (buffer *packetBuffer) ReadWithAttributes(payload []byte, attributes packetio.Attributes) (int, packetio.Attributes, error) { + return buffer.Buffer.Read(payload, attributes) +} + +func (buffer *packetBuffer) WriteWithAttributes(payload []byte, attributes packetio.Attributes) (int, error) { + return buffer.Buffer.Write(payload, attributes) +} diff --git a/internal/media/forwarding/packet_buffer_test.go b/internal/media/forwarding/packet_buffer_test.go new file mode 100644 index 000000000..84096224d --- /dev/null +++ b/internal/media/forwarding/packet_buffer_test.go @@ -0,0 +1,43 @@ +package forwarding + +import ( + "errors" + "io" + "testing" + "time" + + "github.com/pion/transport/v5/packetio" +) + +func TestPacketBufferRetainsSRTPContract(t *testing.T) { + buffer := &packetBuffer{Buffer: packetio.NewBuffer()} + buffer.SetLimitCount(2) + if err := buffer.SetReadDeadline(time.Now().Add(time.Second)); err != nil { + t.Fatal(err) + } + var stream io.ReadWriteCloser = buffer + t.Cleanup(func() { _ = stream.Close() }) + if _, err := stream.Write([]byte{1}); err != nil { + t.Fatal(err) + } + if _, err := buffer.WriteWithAttributes([]byte{2}, packetio.Attributes{{Key: "arrival", Value: 7}}); err != nil { + t.Fatal(err) + } + if _, err := stream.Write([]byte{3}); !errors.Is(err, packetio.ErrFull) { + t.Fatalf("queue limit lost at SRTP boundary: %v", err) + } + payload := make([]byte, 8) + if n, err := stream.Read(payload); err != nil || n != 1 || payload[0] != 1 { + t.Fatalf("ordinary SRTP read: n=%d err=%v", n, err) + } + n, attributes, err := buffer.ReadWithAttributes(payload, nil) + if err != nil || n != 1 || payload[0] != 2 || attributes.Get("arrival") != 7 { + t.Fatalf("packet or attributes lost: n=%d attributes=%v err=%v", n, attributes, err) + } + if err := stream.Close(); err != nil { + t.Fatal(err) + } + if _, err := stream.Read(payload); !errors.Is(err, io.EOF) { + t.Fatalf("closed SRTP buffer did not finish: %v", err) + } +} diff --git a/internal/media/forwarding/publication.go b/internal/media/forwarding/publication.go index 08537672a..84c1922c6 100644 --- a/internal/media/forwarding/publication.go +++ b/internal/media/forwarding/publication.go @@ -23,7 +23,7 @@ import ( "github.com/pion/interceptor" "github.com/pion/rtcp" "github.com/pion/rtp" - "github.com/pion/transport/v4/packetio" + "github.com/pion/transport/v5/packetio" "github.com/pion/webrtc/v4" ) @@ -110,7 +110,7 @@ func NewPublication(options TransportOptions) (_ *Publication, err error) { return reader } } - input := packetio.NewBuffer() + input := &packetBuffer{Buffer: packetio.NewBuffer()} limit := 1_000_000 if kind == packetio.RTCPBufferPacket { limit = 100_000 diff --git a/internal/media/forwarding/recovery_test.go b/internal/media/forwarding/recovery_test.go index 6edcae045..733ad6ede 100644 --- a/internal/media/forwarding/recovery_test.go +++ b/internal/media/forwarding/recovery_test.go @@ -17,7 +17,7 @@ import ( "github.com/pion/ice/v4" "github.com/pion/interceptor" "github.com/pion/logging" - "github.com/pion/transport/v4/vnet" + "github.com/pion/transport/v5/vnet" "github.com/pion/webrtc/v4" rtmedia "github.com/pion/webrtc/v4/pkg/media" "github.com/thesyncim/gopus" diff --git a/internal/media/forwarding/transport.go b/internal/media/forwarding/transport.go index eb33fed90..2f872a9a6 100644 --- a/internal/media/forwarding/transport.go +++ b/internal/media/forwarding/transport.go @@ -23,7 +23,7 @@ import ( "github.com/pion/interceptor" "github.com/pion/rtcp" "github.com/pion/rtp" - "github.com/pion/transport/v4/packetio" + "github.com/pion/transport/v5/packetio" "github.com/pion/webrtc/v4" ) @@ -102,7 +102,7 @@ func NewTransport(options TransportOptions) (_ *Transport, err error) { } } // Preserve stock Pion SRTP/SRTCP bounds for unclaimed streams, including audio. - input := packetio.NewBuffer() + input := &packetBuffer{Buffer: packetio.NewBuffer()} limit := 1_000_000 if kind == packetio.RTCPBufferPacket { limit = 100_000 diff --git a/internal/server/stun/server_test.go b/internal/server/stun/server_test.go index 958f644ad..0c620bc69 100644 --- a/internal/server/stun/server_test.go +++ b/internal/server/stun/server_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/pion/stun/v3" + "github.com/pion/stun/v4" ) func TestThreeSTUNListenersWithoutRelayAndWithCompleteShutdown(t *testing.T) { From 091df10fa814ceb4def09230540bc930956c35c6 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:58:20 +0800 Subject: [PATCH 142/262] docs(network): assess browser handshake latency proposals --- docs/research/nat-traversal.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index e6f7f266e..6ac18b01e 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -386,6 +386,16 @@ against that same endpoint pair and relay policy. interest does not establish interoperable product support. Evaluate only framework-owned negotiation and stock Browser builds, retaining authentication and ordinary fallback. A Browser field-trial flag is not an adoption path. +- The July 2026 [WARP draft](https://datatracker.ietf.org/doc/html/draft-uberti-tsvwg-warp-00) + combines SPED, SCTP setup negotiation (SNAP) and DTLS 1.3. Its combined model + reduces media-and-data setup from six round trips to two; media alone starts + after four round trips in its baseline. These are handshake savings, not + evidence that previously unreachable NAT pairs connect. Pion's + [ICE v5 release candidate](https://github.com/pion/ice/releases/tag/v5.0.0-rc.6) + already contains SPED/DTLS 1.3 work, so an expired individual draft does not + imply abandoned implementation. Follow stable framework support and negotiated + stock-Browser interoperability before a latency comparison; this assessment + does not accept a prerelease transport upgrade or changes to ICE consent. #### Stable Pion Interoperability From 55f5f6be8b29c09497ee6bbf8a921dbc65276112 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:12:52 +0800 Subject: [PATCH 143/262] fix(ui): align media input drafts and responsive settings --- .../components/living/QualityPresets.tsx | 71 +++++++++++++-- src/client/pages/ControlsPreview.tsx | 9 +- src/client/pages/HostPage.tsx | 87 +------------------ src/client/styles.css | 4 +- 4 files changed, 78 insertions(+), 93 deletions(-) diff --git a/src/client/components/living/QualityPresets.tsx b/src/client/components/living/QualityPresets.tsx index 7765e3e59..289147fda 100644 --- a/src/client/components/living/QualityPresets.tsx +++ b/src/client/components/living/QualityPresets.tsx @@ -1,10 +1,13 @@ -import { Fragment } from "react"; +import { Fragment, useRef } from "react"; +import { MIN_AUDIO_BITRATE } from "../../../shared/protocol"; import { QUALITY_PROFILES, QUALITY_PROFILE_KEYS, QUALITY_RESOLUTIONS, qualityResolutionEqual, captureFramerateCeiling, VIDEO_CAPTURE_LIMITS, type VideoCaptureLimits, type QualityProfileId, type QualityResolution, + SCREEN_AUDIO_BITRATES, type ScreenAudioPreset, } from "../../media/quality"; -import { useCopy } from "../../ui/copy"; -import { Chip } from "./primitives"; +import { useCopy, type CopyKey } from "../../ui/copy"; +import { Glyph } from "../../ui/icons"; +import { Cap, Chip } from "./primitives"; import { QualityTileGlyph } from "./QualityTileGlyph"; import { Tooltip } from "./Tooltip"; @@ -40,6 +43,7 @@ export function ResolutionControl({ value, limits, disabled, onChange }: { onChange: (resolution: QualityResolution) => void; }) { const { t, vis } = useCopy(); + const controls = useRef(null); const axes = ["width", "height"] as const; const reset = (container: HTMLElement) => { for (const axis of axes) container.querySelector(`input[name="${axis}"]`)!.value = String(value[axis]); @@ -58,7 +62,7 @@ export function ResolutionControl({ value, limits, disabled, onChange }: { } if (!qualityResolutionEqual(value, next)) onChange(next); }; - return
{ if (!event.currentTarget.contains(event.relatedTarget)) apply(event.currentTarget); }} @@ -73,7 +77,11 @@ export function ResolutionControl({ value, limits, disabled, onChange }: { {Object.entries(QUALITY_RESOLUTIONS).map(([label, resolution]) => onChange(resolution)}>{label})} + title={label} hint="hint-quality" onClick={() => { + // Selecting even the current preset replaces an unsubmitted draft. + reset(controls.current!); + onChange(resolution); + }}>{label})}
; } + +const AUDIO_QUALITY_CAPTIONS: Record = { + saver: "host.advanced.audio.saver", + music: "host.advanced.audio.music", + "very-high": "host.advanced.audio.veryHigh", +}; + +export function AudioBitrateControl({ value, max, disabled, onChange }: { + value: number; + max: number; + disabled: boolean; + onChange: (bitrate: number) => void; +}) { + const { t, vis } = useCopy(); + return
+ +
+ {(Object.keys(SCREEN_AUDIO_BITRATES) as ScreenAudioPreset[]).map(preset => { + const bitrate = SCREEN_AUDIO_BITRATES[preset]; + return max} + title={t("host.audio.title", { label: t(AUDIO_QUALITY_CAPTIONS[preset]), kbps: String(bitrate / 1_000) })} + hint="hint-audio-quality" onClick={() => onChange(bitrate)}> + {bitrate / 1_000} + ; + })} +
+ + + +
; +} diff --git a/src/client/pages/ControlsPreview.tsx b/src/client/pages/ControlsPreview.tsx index c639369b2..88380015d 100644 --- a/src/client/pages/ControlsPreview.tsx +++ b/src/client/pages/ControlsPreview.tsx @@ -19,9 +19,9 @@ import { LauncherForm, type AppMode } from "../components/living/LauncherForm"; import { AppReleaseNotice } from "../components/living/AppReleaseNotice"; import { useTheaterMode } from "../components/living/use-theater-mode"; -import { QualityPresets, ResolutionControl } from "../components/living/QualityPresets"; +import { AudioBitrateControl, QualityPresets, ResolutionControl } from "../components/living/QualityPresets"; import { RoomCodeInput } from "../components/living/RoomCodeInput"; -import { VIDEO_CAPTURE_LIMITS, QUALITY_PROFILES, type QualityProfileId, type QualityResolution } from "../media/quality"; +import { MAX_SCREEN_AUDIO_BITRATE, VIDEO_CAPTURE_LIMITS, QUALITY_PROFILES, type QualityProfileId, type QualityResolution } from "../media/quality"; import type { loadCameraPreviews } from "../media/camera-previews"; import { Tooltip } from "../components/living/Tooltip"; import { StatusIndicator } from "../components/living/StatusIndicator"; @@ -58,6 +58,7 @@ export function ControlsPreview() { const [microphoneDevice, setMicrophoneDevice] = useState(""); const [sharingSettings, setSharingSettings] = useState(false); const [resolution, setResolution] = useState(QUALITY_PROFILES["1080p30"].resolution); + const [audioBitrate, setAudioBitrate] = useState(QUALITY_PROFILES["1080p30"].maxAudioBitrate); const [launchMode, setLaunchMode] = useState("link"); const [majorUpdate, setMajorUpdate] = useState(true); const releaseVersion = majorUpdate ? "v2.0.0" : "v1.1.0"; @@ -127,7 +128,9 @@ export function ControlsPreview() {
} + voiceProcessing={voiceProcessing} onVoiceProcessing={setVoiceProcessing} /> + } />
diff --git a/src/client/pages/HostPage.tsx b/src/client/pages/HostPage.tsx index 8991adcc4..a305eaccd 100644 --- a/src/client/pages/HostPage.tsx +++ b/src/client/pages/HostPage.tsx @@ -10,7 +10,6 @@ import { import { DEFAULT_ROUTE_POLICY, MAX_VIEWER_PASSWORD_LENGTH, - MIN_AUDIO_BITRATE, viewerPasswordSchema, type CreateRoomResponse, type ClientMessage, @@ -53,7 +52,7 @@ import type { ComicKind } from "../components/living/Comic"; import type { HintKind } from "../ui/visual-kinds"; import type { ComicTone } from "../components/living/comic-presentation"; import { Tooltip } from "../components/living/Tooltip"; -import { QualityPresets, ResolutionControl } from "../components/living/QualityPresets"; +import { AudioBitrateControl, QualityPresets, ResolutionControl } from "../components/living/QualityPresets"; import { RoomCodeInput, RoomCodeError } from "../components/living/RoomCodeInput"; import { MetricCell } from "../components/living/Metrics"; import { @@ -135,12 +134,10 @@ import { qualitySettingsLabel, screenAudioBitrate, MAX_SCREEN_AUDIO_BITRATE, - SCREEN_AUDIO_BITRATES, setMediaPaused, videoQualitySettingsEqual, type DegradationPreference, type QualitySettings, - type ScreenAudioPreset, } from "../media/quality"; import { HostSfuRoute } from "../media/host-sfu-route"; import { BrowserEncodingPool } from "../media/browser-encoding-pool"; @@ -236,12 +233,6 @@ const PREFERENCE_PRESENTATION: Record< }, }; -const AUDIO_QUALITY_CAPTIONS: Record = { - saver: "host.advanced.audio.saver", - music: "host.advanced.audio.music", - "very-high": "host.advanced.audio.veryHigh", -}; - type ViewerQualityEvidence = Extract< ServerMessage, { type: "viewer-quality-evidence" } @@ -3992,79 +3983,9 @@ export function HostPage({ onVoiceProcessing={enabled => void changeMicrophone(microphoneEnabled, microphoneDevices.browser, enabled)} /> ) : null} -
- - - - -
- {( - Object.keys( - AUDIO_QUALITY_CAPTIONS, - ) as ScreenAudioPreset[] - ).map((audioQuality) => ( - audioBitrateCeiling} - title={t("host.audio.title", { - label: t(AUDIO_QUALITY_CAPTIONS[audioQuality]), - kbps: String( - SCREEN_AUDIO_BITRATES[audioQuality] / 1_000, - ), - })} - hint="hint-audio-quality" - onClick={() => changeMaxAudioBitrate(SCREEN_AUDIO_BITRATES[audioQuality])} - > - - - {SCREEN_AUDIO_BITRATES[audioQuality] / 1_000} - - - ))} -
- - - -
+ } technical={<>
diff --git a/src/client/styles.css b/src/client/styles.css index fa225d2ac..7fd85d532 100644 --- a/src/client/styles.css +++ b/src/client/styles.css @@ -1822,7 +1822,7 @@ html[data-theme="dark"] .lr-switch::after { background: var(--ink); } margin: 10px auto 0; } .lr-stage-notices:empty { display: none; } -.lr-sharing-settings { width: min(820px, 100%); margin: 0 auto; } +.lr-sharing-settings { width: min(820px, 100%); margin: 0 auto; container: sharing-settings / inline-size; } .lr-sharing-panel { margin-top: 14px; padding: 22px; background: var(--paper); border: 1px solid var(--control-edge); border-radius: 24px; box-shadow: var(--control-shadow); @@ -1851,7 +1851,7 @@ html[data-theme="dark"] .lr-switch::after { background: var(--ink); } .lr-sharing-technical > div { display: grid; gap: 18px; padding-top: 16px; } @keyframes lr-sharing-enter { from { opacity: 0; transform: translateY(-6px); } to { opacity: 1; transform: translateY(0); } } @media (prefers-reduced-motion: reduce) { .lr-sharing-panel { animation: none; } } -@media (max-width: 680px) { +@container sharing-settings (max-width: 680px) { .lr-sharing-panel { padding: 16px; border-radius: 20px; } .lr-sharing-columns { grid-template-columns: 1fr; gap: 22px; } .lr-sharing-limits { grid-template-columns: 1fr; } From 82e3f253935cf4d74b5c0773dbf074e9858d7a54 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:29:49 +0800 Subject: [PATCH 144/262] fix(media): preserve macOS content geometry within output limits --- docs/research/native-client-media.md | 24 ++- docs/status.md | 3 +- docs/todo.md | 6 +- native/capture/darwin/README.md | 10 +- native/capture/darwin/main.swift | 270 ++++++++++++++++++++------- scripts/check-go.mjs | 1 + 6 files changed, 230 insertions(+), 84 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index da70412a5..614f16f4b 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -547,14 +547,22 @@ one pipeline, checks the output ceiling and image corners, and exercises the smallest derived output. Compilation and those regressions passed on Linux; Portal capture and hardware encoder resizing still need a physical device. -macOS remains a concrete implementation gap: ScreenCaptureKit and VideoToolbox -currently use the requested dimensions as a fixed canvas. The capture adapter -also ignores per-frame content geometry. Apple's -[capture sample](https://developer.apple.com/documentation/screencapturekit/capturing-screen-content-in-macos) -distinguishes buffer dimensions, content rectangle and pixel scale; changing only -the initial window size would not cover live resizing or Retina scale changes. -Review that boundary under the existing capture/encoder owners, then verify -portrait, small-source and resizing behavior with the macOS SDK and hardware. +The macOS candidate now interprets ScreenCaptureKit's per-frame content rectangle, +pixel scale and content scale, following Apple's +[frame-metadata guidance](https://developer.apple.com/videos/play/wwdc2022/10155/). +It retains the bounded capture surface, describes its content through CoreVideo's +clean aperture and uses VideoToolbox's +[source-aperture scaling](https://developer.apple.com/documentation/videotoolbox/kvtscalingmode_cropsourcetocleanaperture). +Each existing output worker fits the unscaled source within its ceiling and +replaces only its compression session when the fitted size changes. The frame +envelope carries actual dimensions; declared output limits and room/route +identity remain unchanged. Encoded-input derivation uses the same worker. + +This implementation has not yet been compiled with the macOS SDK or executed on +macOS. The existing native check now includes content-geometry validation and +extends hardware self-testing across successive source shapes. Run those checks +on Apple silicon and Intel, then verify real window resizing, Retina transitions, +cropped image fidelity and quiet-source recovery before accepting this boundary. Local synthetic-input controls exercised that production worker through four successive source sizes: 1080x1920, 3440x1440, 640x480 and 1920x1080. Under the diff --git a/docs/status.md b/docs/status.md index 8e2fd55fe..5f550cea1 100644 --- a/docs/status.md +++ b/docs/status.md @@ -70,7 +70,8 @@ Custom width/height controls apply combined size and frame-rate limits, admittin 4K30. Windows and Linux output preserve source shape within the selected ceiling. The [media assessment](./research/native-client-media.md#media-capability-extension-assessment) owns local codec, derived-output, SFU, Browser/Native fanout and live-control -evidence. macOS geometry, cross-browser/device and route-change acceptance, and +evidence. The macOS geometry implementation still needs SDK compilation and +hardware verification; cross-browser/device and route-change acceptance, and additional codecs remain open. Magicsock adoption is no-go for this candidate: its App-to-App augmentation does not improve the core App-to-Browser use case. The diff --git a/docs/todo.md b/docs/todo.md index cfb17f607..59d5b6d57 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -61,9 +61,9 @@ Implementation needs a concrete useful result within the product model. Extend Browser interoperability and mixed-topology checks to limited receivers and physical camera orientation behavior. Browser display capture, Browser relay and Browser-to-Native fanout now have - bounded high-resolution evidence. macOS geometry and physical 4K/game capture - remain open. Preserve safety - limits and framework-owned adaptation. The + bounded high-resolution evidence. macOS geometry SDK/hardware checks and + physical 4K/game capture remain open. Preserve safety limits and + framework-owned adaptation. The [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment) retain explicit receive-limit enforcement and the current 320 kbps sender ceiling. Physical listening, mobile and field-network acceptance remain open; diff --git a/native/capture/darwin/README.md b/native/capture/darwin/README.md index 610f63522..f0600fa1c 100644 --- a/native/capture/darwin/README.md +++ b/native/capture/darwin/README.md @@ -14,13 +14,17 @@ machine's architecture: ```sh sh native/capture/darwin/build.sh /outside/repository/build +/outside/repository/build/piik-capture --geometry-self-test /outside/repository/build/piik-capture --self-test /outside/repository/build/piik-capture --probe ``` -`--self-test` encodes one in-memory 420v frame through the same hardware-only -VideoToolbox path and requires a constrained-baseline SPS/PPS/IDR. It needs no -Screen Recording permission. Real source selection, capture, and recovery still +`--geometry-self-test` checks content cropping, Retina scaling and rejection of +invalid frame geometry without requiring a hardware encoder. `--self-test` +encodes successive landscape, portrait, ultrawide and small 420v buffers through +one output worker, requiring the fitted dimensions and a constrained-baseline +SPS/PPS/IDR after each size change. Neither needs Screen Recording permission. +Real source selection, window resizing, display-scale changes and recovery still require a physical macOS gate. The first source-list or capture request is subject to the normal macOS Screen diff --git a/native/capture/darwin/main.swift b/native/capture/darwin/main.swift index 6e05e6a3a..716fa1724 100644 --- a/native/capture/darwin/main.swift +++ b/native/capture/darwin/main.swift @@ -47,6 +47,61 @@ private struct OutputProfile: Codable { var frameInterval: CMTime { CMTime(value: 1, timescale: fps) } } +// Keep the captured pixels, their clean aperture and their unscaled size together. +// ScreenCaptureKit may place scaled window content inside a larger fixed surface. +private struct VideoImage { + let buffer: CVPixelBuffer + let size: CGSize + + init(_ buffer: CVPixelBuffer) { + self.buffer = buffer + self.size = CVImageBufferGetCleanRect(buffer).size + } + + init?(_ buffer: CVPixelBuffer, attachments: [SCStreamFrameInfo: Any]) { + guard let dictionary = attachments[.contentRect] as? [String: Any], + let rect = CGRect(dictionaryRepresentation: dictionary as CFDictionary), + let scale = attachments[.scaleFactor] as? CGFloat, + let contentScale = attachments[.contentScale] as? CGFloat, + scale.isFinite, scale > 0, contentScale.isFinite, contentScale > 0 else { + return nil + } + let crop = CGRect(x: rect.minX * scale, y: rect.minY * scale, + width: rect.width * scale, height: rect.height * scale) + let bounds = CGRect(x: 0, y: 0, width: CVPixelBufferGetWidth(buffer), + height: CVPixelBufferGetHeight(buffer)) + guard crop.minX.isFinite, crop.minY.isFinite, + crop.width.isFinite, crop.height.isFinite, + crop.width > 0, crop.height > 0, bounds.contains(crop) else { return nil } + let size = CGSize(width: crop.width / contentScale, height: crop.height / contentScale) + guard size.width.isFinite, size.height.isFinite else { return nil } + // Set once before sharing the retained buffer with the output workers. + let aperture: [String: Any] = [ + kCVImageBufferCleanApertureWidthKey as String: crop.width, + kCVImageBufferCleanApertureHeightKey as String: crop.height, + kCVImageBufferCleanApertureHorizontalOffsetKey as String: crop.midX - bounds.midX, + kCVImageBufferCleanApertureVerticalOffsetKey as String: crop.midY - bounds.midY, + ] + CVBufferSetAttachment(buffer, kCVImageBufferCleanApertureKey, + aperture as CFDictionary, .shouldPropagate) + self.buffer = buffer + self.size = size + } + + func fitted(to profile: OutputProfile) throws -> OutputProfile { + guard size.width.isFinite, size.height.isFinite, size.width > 0, size.height > 0 else { + throw CaptureFailure(description: "invalid video content size") + } + let scale = min(1, CGFloat(profile.width) / size.width, + CGFloat(profile.height) / size.height) + return OutputProfile( + width: max(2, Int((size.width * scale / 2).rounded(.down)) * 2), + height: max(2, Int((size.height * scale / 2).rounded(.down)) * 2), + fps: profile.fps, bitrate: profile.bitrate + ) + } +} + private enum CaptureControl { case keyFrame(Int) case active(Int, Bool) @@ -220,21 +275,8 @@ private func shareableWindows(_ content: SCShareableContent) -> [(SCWindow, Capt } } -private func hardwareH264Available() -> Bool { - do { - let encoder = try HardwareEncoder( - writer: ProtocolWriter(), - done: StopSignal() - ) - encoder.close() - return true - } catch { - return false - } -} - private func probe() throws { - let encoders = hardwareH264Available() + let encoders = HardwareEncoder.isAvailable() ? [EncoderProbe( index: 0, name: "VideoToolbox H.264", @@ -522,7 +564,7 @@ private func h264ProfileLevelID(_ payload: Data) -> String? { private final class HardwareEncoder { private struct Input { - let image: CVImageBuffer + let image: VideoImage let timestamp: CMTime } @@ -542,9 +584,11 @@ private final class HardwareEncoder { private var currentBitrate: Int private var lastFrameBucket: Int64? private let sourceFrameRate: Int32 + private let preference: DegradationPreference private let profile: OutputProfile + private var output: OutputProfile? private let layer: UInt8 - private let onActive: (String) throws -> Void + private let onActive: (String, OutputProfile) throws -> Void private let onFailure: (Error) -> Void init( @@ -553,9 +597,9 @@ private final class HardwareEncoder { profile sourceProfile: VideoProfile = defaultVideoProfile, output: OutputProfile? = nil, layer: UInt8 = 0, - onActive: @escaping (String) throws -> Void = { _ in }, + onActive: @escaping (String, OutputProfile) throws -> Void = { _, _ in }, onFailure: ((Error) -> Void)? = nil - ) throws { + ) { self.writer = writer let profile = output ?? OutputProfile( width: sourceProfile.width, height: sourceProfile.height, @@ -563,12 +607,40 @@ private final class HardwareEncoder { ) self.profile = profile self.sourceFrameRate = sourceProfile.frameRate + self.preference = sourceProfile.preference self.layer = layer self.onActive = onActive self.onFailure = onFailure ?? { done.signal($0) } self.queue = DispatchQueue(label: "piik.capture.encoder.\(layer)") desiredBitrate = profile.bitrate currentBitrate = profile.bitrate + } + + static func isAvailable() -> Bool { + let encoder = HardwareEncoder(writer: ProtocolWriter(), done: StopSignal()) + defer { encoder.close() } + do { + _ = try encoder.queue.sync { + try encoder.prepare(encoder.profile, bitrate: encoder.profile.bitrate) + } + return true + } catch { + return false + } + } + + // Called only by the encoder queue after the previous frame has completed. + // A size change replaces this output's codec, not its capture or media route. + private func prepare(_ requested: OutputProfile, bitrate: Int) throws -> Bool { + if let output, output.width == requested.width, output.height == requested.height { + return false + } + if let session { VTCompressionSessionInvalidate(session) } + if let transfer { VTPixelTransferSessionInvalidate(transfer) } + session = nil + transfer = nil + output = nil + let profile = requested let specification = [ kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder as String: true, ] as CFDictionary @@ -603,20 +675,32 @@ private final class HardwareEncoder { try set(kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse) try set(kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_ConstrainedBaseline_AutoLevel) - try set(kVTCompressionPropertyKey_AverageBitRate, NSNumber(value: profile.bitrate)) + try set(kVTCompressionPropertyKey_AverageBitRate, NSNumber(value: bitrate)) try set(kVTCompressionPropertyKey_ExpectedFrameRate, NSNumber(value: profile.fps)) try set(kVTCompressionPropertyKey_MaxKeyFrameInterval, NSNumber(value: profile.fps * 2)) - if sourceProfile.preference != .balanced { + if preference != .balanced { try? set( kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, - sourceProfile.preference == .framerate ? kCFBooleanTrue : kCFBooleanFalse + preference == .framerate ? kCFBooleanTrue : kCFBooleanFalse ) } try require(VTPixelTransferSessionCreate( allocator: nil, pixelTransferSessionOut: &transfer ), "videotoolbox-scaler-create") + guard let transfer else { + throw CaptureFailure(description: "VideoToolbox returned no scaler") + } + try require(VTSessionSetProperty(transfer, + key: kVTPixelTransferPropertyKey_ScalingMode, + value: kVTScalingMode_CropSourceToCleanAperture), "videotoolbox-scale-mode") try require(VTCompressionSessionPrepareToEncodeFrames(created), "videotoolbox-prepare") + output = profile + currentBitrate = bitrate + lock.lock() + decodable = false + lock.unlock() + return true } catch { VTCompressionSessionInvalidate(created) if let transfer { VTPixelTransferSessionInvalidate(transfer) } @@ -646,7 +730,7 @@ private final class HardwareEncoder { lock.unlock() } - func encode(_ image: CVImageBuffer, timestamp: CMTime) { + func encode(_ image: VideoImage, timestamp: CMTime) { lock.lock() guard !closed, !failed else { lock.unlock() @@ -678,18 +762,22 @@ private final class HardwareEncoder { return } pending = nil - let force = forceKeyFrame + var force = forceKeyFrame forceKeyFrame = false let bitrate = desiredBitrate encodingTimestamp = input.timestamp lock.unlock() - guard let session else { return } do { + let output = try input.image.fitted(to: profile) + if try prepare(output, bitrate: bitrate) { force = true } + guard let session else { + throw CaptureFailure(description: "VideoToolbox encoder is unavailable") + } if currentBitrate != bitrate { try set(kVTCompressionPropertyKey_AverageBitRate, NSNumber(value: bitrate)) currentBitrate = bitrate } - let image = try scaled(input.image) + let image = try scaled(input.image.buffer, output: output) let properties = force ? [kVTEncodeFrameOptionKey_ForceKeyFrame as String: true] as CFDictionary : nil @@ -731,7 +819,7 @@ private final class HardwareEncoder { throw CaptureFailure(description: "invalid H.264 timestamp") } try writer.writeH264( - payload, keyFrame: keyFrame, layer: layer, profile: profile, + payload, keyFrame: keyFrame, layer: layer, profile: output, timestamp: UInt64(CMTimeConvertScale( pts, timescale: 10_000_000, method: .roundTowardZero ).value), @@ -741,7 +829,7 @@ private final class HardwareEncoder { guard let profileLevelId = h264ProfileLevelID(payload) else { throw CaptureFailure(description: "H.264 key frame has no SPS profile") } - try onActive(profileLevelId) + try onActive(profileLevelId, output) } } catch { if force { requestKeyFrame() } @@ -769,9 +857,11 @@ private final class HardwareEncoder { if current { queue.async { self.encodePending() } } } - private func scaled(_ image: CVImageBuffer) throws -> CVPixelBuffer { - if CVPixelBufferGetWidth(image) == profile.width && - CVPixelBufferGetHeight(image) == profile.height { + private func scaled(_ image: CVImageBuffer, output: OutputProfile) throws -> CVPixelBuffer { + let bounds = CGRect(x: 0, y: 0, width: CVPixelBufferGetWidth(image), + height: CVPixelBufferGetHeight(image)) + if Int(bounds.width) == output.width && Int(bounds.height) == output.height && + CVImageBufferGetCleanRect(image) == bounds { return image } guard let session, let transfer, @@ -857,22 +947,18 @@ private final class EncoderGroup { try writer.writeStatus(StartingStatus(outputs: outputs)) if !encoded { for index in 0...originalOutput { setActive(index, true) } - if encoders[originalOutput] == nil { - close() - throw CaptureFailure(description: "original hardware video output is unavailable") - } } } - private func reportActive(_ profileLevelId: String) throws { + private func reportActive(_ profileLevelId: String, output: OutputProfile) throws { lock.lock() let publish = !active active = true lock.unlock() if publish { try writer.writeStatus(ActiveStatus( - profileLevelId: profileLevelId, width: profile.width, - height: profile.height, fps: profile.frameRate, outputs: outputs + profileLevelId: profileLevelId, width: output.width, + height: output.height, fps: profile.frameRate, outputs: outputs )) } } @@ -890,7 +976,7 @@ private final class EncoderGroup { if allFailed || startupFailed { done.signal(error) } } - func encode(_ image: CVImageBuffer, timestamp: CMTime) throws { + func encode(_ image: VideoImage, timestamp: CMTime) throws { lifecycle.lock() defer { lifecycle.unlock() } if closed { return } @@ -928,21 +1014,17 @@ private final class EncoderGroup { lock.lock() failedLayers.remove(layer) lock.unlock() - do { - let encoder = try HardwareEncoder( - writer: writer, done: done, profile: profile, - output: outputs[index], layer: layer, - onActive: { [weak self] profileLevelId in - guard let self, index == self.originalOutput else { return } - try self.reportActive(profileLevelId) - }, - onFailure: { [weak self] in self?.unavailable(layer, $0) } - ) - encoder.setBitrate(bitrates[index]) - encoders[index] = encoder - } catch { - unavailable(layer, error) - } + let encoder = HardwareEncoder( + writer: writer, done: done, profile: profile, + output: outputs[index], layer: layer, + onActive: { [weak self] profileLevelId, output in + guard let self, index == self.originalOutput else { return } + try self.reportActive(profileLevelId, output: output) + }, + onFailure: { [weak self] in self?.unavailable(layer, $0) } + ) + encoder.setBitrate(bitrates[index]) + encoders[index] = encoder } func setBitrate(_ layer: Int, _ bitrate: Int) { @@ -978,7 +1060,9 @@ private final class EnvelopeCollector { } } -private func syntheticPixelBuffer() throws -> CVPixelBuffer { +private func syntheticPixelBuffer( + width imageWidth: Int = width, height imageHeight: Int = height +) throws -> CVPixelBuffer { let attributes = [ kCVPixelBufferIOSurfacePropertiesKey as String: [:] as CFDictionary, ] as CFDictionary @@ -986,8 +1070,8 @@ private func syntheticPixelBuffer() throws -> CVPixelBuffer { try require( CVPixelBufferCreate( kCFAllocatorDefault, - width, - height, + imageWidth, + imageHeight, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, attributes, &value @@ -1032,15 +1116,57 @@ private func containsRequiredH264Units(_ envelope: Data) -> Bool { private func selfTest() throws { let collector = EnvelopeCollector() let writer = ProtocolWriter(output: { collector.append($0) }) - let encoder = try HardwareEncoder(writer: writer, done: StopSignal()) - encoder.encode( - try syntheticPixelBuffer(), - timestamp: CMTime(value: 1, timescale: frameRate) - ) - encoder.flush() - encoder.close() - guard collector.snapshot().contains(where: containsRequiredH264Units) else { - throw CaptureFailure(description: "hardware H.264 self-test produced no IDR") + let encoder = HardwareEncoder(writer: writer, done: StopSignal()) + defer { encoder.close() } + let cases = [(1280, 720, 1280, 720), (720, 1280, 404, 720), + (1720, 720, 1280, 534), (640, 480, 640, 480), (1280, 720, 1280, 720)] + for (index, item) in cases.enumerated() { + let previous = collector.snapshot().count + encoder.encode( + VideoImage(try syntheticPixelBuffer(width: item.0, height: item.1)), + timestamp: CMTime(value: Int64(index + 1), timescale: frameRate) + ) + encoder.flush() + guard let envelope = collector.snapshot().dropFirst(previous).first(where: containsRequiredH264Units), + Int(envelope[24]) * 256 + Int(envelope[25]) == item.2, + Int(envelope[26]) * 256 + Int(envelope[27]) == item.3 else { + throw CaptureFailure(description: "hardware H.264 resize self-test failed at \(index)") + } + } +} + +private func geometrySelfTest() throws { + let buffer = try syntheticPixelBuffer() + let ceiling = OutputProfile(width: 1280, height: 720, fps: frameRate, bitrate: bitrate) + let rect = CGRect(x: 50, y: 40, width: 200, height: 300) + let attachments: [SCStreamFrameInfo: Any] = [ + .contentRect: rect.dictionaryRepresentation, + .scaleFactor: CGFloat(2), .contentScale: CGFloat(1), + ] + guard let image = VideoImage(buffer, attachments: attachments), + try image.fitted(to: ceiling).width == 400, + try image.fitted(to: ceiling).height == 600, + CVImageBufferGetCleanRect(buffer) == CGRect(x: 100, y: 80, width: 400, height: 600) else { + throw CaptureFailure(description: "capture content geometry self-test failed") + } + var upscaled = attachments + upscaled[.contentScale] = CGFloat(2) + guard let smaller = VideoImage(buffer, attachments: upscaled), + try smaller.fitted(to: ceiling).width == 200, + try smaller.fitted(to: ceiling).height == 300 else { + throw CaptureFailure(description: "small capture was enlarged") + } + for value in [CGFloat(0), -.infinity, .nan] { + var invalid = attachments + invalid[.contentScale] = value + guard VideoImage(buffer, attachments: invalid) == nil else { + throw CaptureFailure(description: "invalid capture scale was accepted") + } + } + var outside = attachments + outside[.contentRect] = CGRect(x: 700, y: 0, width: 200, height: 300).dictionaryRepresentation + guard VideoImage(buffer, attachments: outside) == nil else { + throw CaptureFailure(description: "out-of-bounds capture content was accepted") } } @@ -1239,7 +1365,7 @@ private final class CaptureOutput: NSObject, SCStreamOutput, SCStreamDelegate { private let done: StopSignal private let queue: DispatchQueue private let profile: VideoProfile - private var lastImage: CVImageBuffer? + private var lastImage: VideoImage? private var lastTimestamp = CMTime.invalid init( @@ -1287,9 +1413,11 @@ private final class CaptureOutput: NSObject, SCStreamOutput, SCStreamDelegate { sampleBuffer, createIfNecessary: false ) as? [[SCStreamFrameInfo: Any]], - let rawStatus = attachments.first?[.status] as? Int, + let attachments = attachments.first, + let rawStatus = attachments[.status] as? Int, SCFrameStatus(rawValue: rawStatus) == .complete, - let image = CMSampleBufferGetImageBuffer(sampleBuffer) else { + let buffer = CMSampleBufferGetImageBuffer(sampleBuffer), + let image = VideoImage(buffer, attachments: attachments) else { return } let capturedTimestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) @@ -1630,7 +1758,7 @@ private final class H264InputDecoder { try require(decodeStatus, "decode-output") guard let image else { throw CaptureFailure(description: "decoder did not produce an image") } lastTimestamp = input.timestamp - try encoders.encode(image, timestamp: input.timestamp) + try encoders.encode(VideoImage(image), timestamp: input.timestamp) } } @@ -1696,6 +1824,8 @@ private func capture(_ arguments: [String]) async throws { let configuration = SCStreamConfiguration() configuration.width = profile.width configuration.height = profile.height + configuration.scalesToFit = true + if #available(macOS 14.0, *) { configuration.preservesAspectRatio = true } configuration.minimumFrameInterval = profile.frameInterval configuration.queueDepth = 5 configuration.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange @@ -1920,6 +2050,8 @@ private struct PiikCapture { try probe() } else if arguments.count == 2, arguments[1] == "--self-test" { try selfTest() + } else if arguments.count == 2, arguments[1] == "--geometry-self-test" { + try geometrySelfTest() } else if arguments.count == 2, arguments[1] == "--list" { try await listSources() } else if arguments.count == 2, arguments[1] == "--list-microphones" { diff --git a/scripts/check-go.mjs b/scripts/check-go.mjs index bb709206a..b84d1b65b 100644 --- a/scripts/check-go.mjs +++ b/scripts/check-go.mjs @@ -193,6 +193,7 @@ function checkPlatformCapture() { throw new Error("Native capture probe returned an invalid contract"); } if (process.platform === "darwin") { + run(executable, ["--geometry-self-test"]); if (probe.adapters.some(adapter => adapter.hardwareH264.length > 0)) { run(executable, ["--self-test"]); } else { From f9ba6d9984f845849709ede71d53dadecd6ccde9 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:48:41 +0800 Subject: [PATCH 145/262] docs(network): separate ICE evidence from retired transport experiment --- docs/research/nat-traversal.md | 1125 ++++------------------ docs/research/node-local-media-design.md | 4 +- 2 files changed, 187 insertions(+), 942 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 6ac18b01e..b6d53fc49 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -336,13 +336,8 @@ hold endpoint pairs, NAT/filtering, IPv6, network transitions and relay policy constant, measuring direct success, relay use and first media separately. No new transport or universal success-rate claim is accepted by this review. -The initial Go control carried Pion ICE/DTLS/SRTP over magicsock and delivered -fresh media after both UDP endpoints changed without new SDP. Loopback and -separate address-and-port-filtering NATs passed five race repetitions. The -fixture supplied peer identities and endpoint updates; it did not prove automatic -OS network-change detection, decoded playback or real-router behavior. Its -seven-pair direct matrix matched Pion within four-second windows. Go reuse is -feasible; product integration and network benefit require the controls below. +The [retired Go experiment](#integration-cost-and-recovery-controls) establishes +reuse feasibility and its cost, not a Browser-compatible reachability gain. ### Browser-Compatible Follow-Up @@ -456,941 +451,191 @@ recovery loop or a Browser fork to enable this option. **Current decision: no-go for the 2.0 candidate.** This composition requires Native App transport at both ends of the enhanced edge. An App Host with a plain Browser Viewer still uses ordinary WebRTC; mixed relaying does not enhance that -Browser edge. The owner therefore stopped integration and field-package work. -Retain the isolated evidence, not a product dependency or a promised feature. -Independently verified ordinary ICE/media repairs remain useful without this -stack. Reopen only for a concrete Browser-compatible benefit or an explicitly -accepted App-only scope; no additional reachable NAT class has been established. - -The isolated `spike/magicsock-assessment` branch retains the comparison probes. -The nested modules are outside Piik's build. Local bootstrap, peer denial and -synthetic SRTP media checks passed. Initial withdrawal checks observed stopped -outbound media; the stronger authorization checks below found revocation defects. -The raw magicsock direct-path matrix found the same reachable classes as fresh -Pion ICE. Retired mapping-cache reproductions remain controls, not the baseline. - -[Tailcat](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/README.md) -provides an upstream Go reuse candidate: magicsock plus WireGuard, with DERP bootstrap -and relay fallback. Its Browser path at that pin is relay-only. Tailscale's open -[Browser P2P proposal](https://github.com/tailscale/tailscale/issues/21027) uses -WebRTC because the Browser cannot expose raw UDP; it is not an implemented -Browser Magicsock hole-puncher. In the isolated -Linux kernel NAT comparison on 2026-10-05, both stacks connected directly through -ordinary/ordinary and open/hard gateways in either direction. Neither delivered -fresh direct payloads through ordinary/hard, hard/ordinary, hard/hard or blocked -UDP pairs within the observation window. Tailcat delivered those cases while -the fixture relay remained available. This demonstrates relay coverage, not -an additional P2P reachability class or a field success rate. - -Relay policy remains a separate integration boundary. Tailcat's documented -fallback may carry media through DERP during startup or recovery. Closing DERP -or counting its packets in a fixture measures the path; neither enforces Piik's -product policy. The reviewed Tailcat API does not expose a direct-only media -policy that retains discovery. Disabling DERP also removes bootstrap/endpoint -exchange, and polling a direct-path status cannot authorize each later packet. - -An isolated packet-policy overlay separates discovery and fixed -[WireGuard handshake messages](https://www.wireguard.com/protocol/#key-exchange-and-data-packets) -from encapsulated data. Blocked UDP carries no media; a relay-allowed peer's -incoming data is also rejected. Removing either direction's guard fails its -corresponding control. The ordinary-NAT Pion fixture delivered fresh direct SRTP -in an initial run but failed cold startup on a repeat, so it is not a passing -acceptance gate. -The relay counts encrypted data envelopes without recording keys or payloads; -ordinary relay delivery validates that counter. The stricter variant that also -blocks WireGuard handshakes leaves inner ICE connecting without media after -20 seconds, even after the outer endpoint is known. Native control replacement, -sibling continuity and final retirement pass with the media-only restriction -on loopback. These checks do not establish Native startup through NAT. - -Without another preparation step, the complete Native Engine/receiver/playback -fixture also fails cold startup through ordinary kernel NAT with this restriction, -before any address move. Its relay-allowed control passes and retains fresh -playback after DERP closes. -Discovery traces show one direction's initial ping unanswered: one endpoint -confirms direct access promptly, while the other confirms it around six seconds -later. Native's eight initial ICE writes end around 1.4 seconds; a pending UDP -flow may still be unaccepted then. The late direct path does not revive that -exhausted generation. The late side can be either endpoint. A one-shot supported -`Client.DiscoPing` before ICE returns the relay result in milliseconds and does -not fix startup. Diagnostic replies and successful key admission therefore -cannot stand in for bidirectional transport preparation. - -The producer/consumer review identifies a bootstrap contract mismatch: -[Tailcat's initial endpoint advertisement](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/tailcat.go#L1570) -uses CallMeMaybe without first sending direct UDP, while -[magicsock's receiver](https://github.com/tailscale/tailscale/blob/a0e471a35b8f/wgengine/magicsock/endpoint.go#L2146) -assumes that send has already opened the peer's stateful firewall. A dependency -candidate follows pings to newly learned endpoints with the existing endpoint -advertisement. Identical advertisements do not cause another reply, and no -timer, retry count or cryptographic check changes. - -With that candidate, three fresh Pion starts deliver direct SRTP within -68–87 ms of their observation start; blocked-UDP and incoming-relay controls -still enforce the media policy. The real Windows VP8 encoder, Native receiver -and actual Chromium/Firefox bridge also retain fresh decoded pictures through -both local-address handoff directions and subsequent DERP closure, within the -unchanged observation bound. The Native fixture counts relayed data from before -bootstrap, not only after direct-path discovery. These are isolated, generated- -media checks; full App/room integration and physical networks remain open. - -A separate TCP preparation control restored startup at about seven seconds of -extra cost but exceeded the spare time in the full handoff check. That extra -flow is not retained; the library bootstrap candidate supersedes it. Fixed-frame -or bare-Pion playback controls still expose the separate encoder and local-bridge -boundaries described below. - -Authentication, key handling, retry timers and Pion budgets remain unchanged. -The experiment narrows the missing direct-media preparation contract; it is not -a supported upstream option, passing adoption gate or improvement to -direct NAT reachability. Product adoption must preserve site/share policy, -including P2P-only and SFU-only rooms, without silently adding another relay. - -Replacing one gateway mapping without notifying endpoints separated recovery -from initial reachability. Neither established direct path delivered within -20 seconds with the relay closed. Fresh Pion discovery/exchange recovered in -about 0.6 seconds after it was started. Tailcat recovered delivery in about -6.5 seconds through its retained relay; closing it immediately lost delivery -again. Keeping traffic and discovery alive for 45 seconds allowed subsequent -direct delivery after relay closure. Both gateway directions were checked. -These timings have different detection/setup boundaries and are not a Piik -startup or playback benchmark. - -The raw Bind adapter also has a composition cost: it consumes unwrapped STUN -payloads, and its discovery retry can outlive Pion's initial check budget. -Dropping the first discovery packet in each direction reproduced that failure; -increasing only the fixture's ICE budget recovered media. The ordinary Pion -loss control recovered without that override. Do not adopt packet prefixes, -diagnostic Ping polling or wider product deadlines as integration shortcuts. - -Keep product integration conditional on a repeatable benefit on matched networks, -with direct coverage, relay coverage, recovery and first picture measured -separately. Go feasibility alone does not establish that benefit. The actual -Native-engine controls below cover gateway mapping, prediction and IPv6; -physical routers and the full mixed Piik topology remain outside the comparison. - -### Isolated Connection-Layer Acceptance - -The 2026-10-07 experiment through `f0ca8056` keeps Pion's ICE/DTLS/SRTP over -Tailcat. Each physical connection owns a virtual UDP tuple and mux; its one-shot -listener retires after admission. Connections to one authorized peer may share -the tunnel. Closing or aborting one flow preserves healthy siblings. Initial -revocation checks stopped outbound media and denied a fresh client; they did not -prove that the old client's inbound traffic was rejected. These checks do not -implement Piik room grants or endpoint exchange. - -Repeated lifecycle and Linux race checks cover overlapping media preparation, -rollback, replacement, canceled preparation followed by retry, bounded datagrams -and blocked-reader retirement. A 350 ms interruption of one flow preserved its -sibling and recovered without replacing SDP. Piik's actual encoded source, -LiveKit forwarder and pacer also carried a small VP8/Opus fixture over ordinary -Pion and Tailcat, including TWCC and keyframe requests. Tailcat recovered an -injected missing video packet through existing NACK handling after DERP closed. -This is forwarding evidence, not throughput or room-controller acceptance. - -A 2026-10-09 inbound-only control found that opening or closing an unrelated -Tailcat listener discards an accepted UDP flow's filter state. The unchanged -control passes; both listener changes stop delivery. Continuous outbound media -can recreate the reverse-flow entry, explaining why bidirectional checks alone -miss this boundary. The isolated repair passes the previous filter to Tailscale's -[existing shared-state API](https://github.com/tailscale/tailscale/blob/a0e471a35b8f/wgengine/filter/filter.go#L218). -Three race repetitions, the copied Tailcat listener suite and the combined -revocation/isolation controls pass. The patch remains experimental; it neither -changes Piik nor fixes the independent inner-ICE recovery failure below. - -Real Browser controls forward generated H.264/Opus and VP8/Opus through Browser -WebRTC → Pion/Tailcat/Pion → Browser WebRTC without re-encoding. Both decode at -640×360 and, with the combined dependency candidate, 1920×1080. The larger input -requests 60 fps with a 6 Mbps ceiling and fixed dimensions; actual encoding -cadence varies, so it establishes neither 1080p60 delivery nor a throughput -advantage. Fresh picture/audio arrive in each two-second window across eight -seconds after DERP closure and stop after revocation. At about 3.6–4.3 Mbps, -reported RTP loss is zero and the largest Tailcat datagram is 1208 bytes, with -no size rejection. This does not bound all RTCP workloads or establish physical -capture, Browser-native Magicsock or complete Piik relay interoperability. - -The 1200-byte Native packetization envelope with TWCC and short MID/RID headers -produced at most 1228-byte encrypted datagrams across Pion's three default SRTP -profiles. Only four bytes remain below Tailcat's 1232-byte ceiling; larger -headers and maximum compound RTCP remain integration checks. The small actual -forwarder fixture does not establish those limits. - -A feedback control found a concrete mismatch: Pion's TWCC recorder generated -a 1244-byte encrypted report and a 1268-byte compound batch. Ordinary Pion -delivered both; the bounded adapter rejected them, and unwrapped Tailcat did -not deliver either within two seconds. A 1144-byte report arrived before and -after. Windows and Linux race checks repeated this result three times each. -Pion ICE treats non-close socket write errors as packet loss, so `WriteRTCP` -returning no error did not prove delivery. This does not establish how often -Piik would produce these batches. Tailcat's -[documented payload bound](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/tailcat.go#L574) -needs a supported feedback-size boundary before integration; reducing RTP -packetization alone does not bound RTCP. No product transport or custom packet -fragmentation was added for this check. -The related [Pion feedback-MTU report](https://github.com/pion/interceptor/issues/416) -remains open as of 2026-10-07. Its out-of-order packet evidence does not establish -the cause of any Piik field report; the pinned recorder has no byte-size option. - -A 2026-10-08 reordering control flushed twelve 100-arrival reports, then supplied -one delayed packet. The recorder included already reported history and generated -a 1248-byte encrypted report from that one new arrival. Shortening the feedback -interval alone cannot bound its size. Windows and three Linux race repetitions -passed the reproduction. The latest tagged -[interceptor v0.1.49 recorder](https://github.com/pion/interceptor/blob/v0.1.49/pkg/twcc/twcc.go) -is byte-identical to the pinned recorder; Tailcat's -[October 7 revision](https://github.com/tailscale/tailcat/blob/52fbad323e9d352f1febb623e0975368e9a8cbdd/tailcat.go#L574) retains -the same payload limit and Tailscale dependency. A dependency bump does not -supply the missing per-report and compound-batch size boundary. - -A local upstream-layer repair candidate now bounds TWCC reports through the -existing recorder split loop and sends them separately before encryption. Its -fixed 1200-byte plaintext budget is experimental, not a supported upstream -option. Unit checks preserve loss, reordered history, wrapped sequence numbers -and arrival times; restoring either old recorder or writer fails its regression -check. A real interceptor-to-Tailcat control delivered all 1500 statuses in two -reports after DERP closed, with a largest encrypted datagram of 1212 bytes. -Windows and three Linux race repetitions passed, as did the upstream TWCC suite -with the candidate. The isolated probe retains the opt-in patch and reproduction; -production dependencies are unchanged. This narrows one integration obstacle, -not the missing direct-reachability benefit or a supported upstream MTU contract. - -The current Go feedback paths were also traced end to end. Pion's default NACK -generator retains a 512-packet window and writes one report per media SSRC; -a full-loss/wrapped-gap control with two SSRCs stays within 156 encrypted bytes. -Normal sender/receiver reports are written separately per stream. Piik's SFU -ingress callback forwards only PLI/FIR, and LiveKit's reachable XR response contains one -DLRR block. No additional oversized producer was found in these paths; this -does not bound arbitrary caller-created RTCP batches or future configurations. - -Kernel queue controls compare established Pion and Tailcat SRTP under delay, -jitter, 5% loss and reordering with DERP closed. Both retained their session and -resumed after a 350 ms blackout in three race repetitions, and after a separate -nine-second blackout. Fresh tags and unique sequences exclude queued/duplicate -success. These are bounded recovery checks, not decoded quality or field coverage. - -Kernel gateway rebinding retained full WebRTC: DERP restored fresh media in about -6.4–6.5 seconds and ICE `connected` in 7.1–8.1 seconds, without new SDP. Its recorded -disconnected interval was about two seconds, inside Piik's three-second grace; -the nine-second blackout reached about four seconds disconnected. Actual Native -Viewer-owner replays cancel grace or request one restart, clear recovery on -connection and retain the receiver. They mock App events, so do not exercise the -restart exchange over Tailcat or prove full integration. With DERP closed, neither -gateway direction recovered within 20 seconds. Fresh media, ICE state and a -unilateral direct address are separate facts. Underlay setup must fit the existing -operation deadline. Local cold setup of about 3.3 seconds for Tailcat versus 0.2 -seconds for Pion disabled STUN/gateway mapping; it is not a WAN startup comparison. - -Actual ICE-restart controls on 2026-10-08 exposed a composition failure. -After a nine-second kernel blackout, ordinary Pion resumed media, but default -Tailcat sometimes exhausted eight inner Binding requests before outer discovery -recovered about 2.4 seconds after restoration. Pion's -[candidate-pair check budget](https://github.com/pion/ice/blob/v4.4.0/agent.go#L790) -and Magicsock's -[rate-limited discovery](https://github.com/tailscale/tailscale/blob/a0e471a35b8f/wgengine/magicsock/endpoint.go#L1421) -are independent: outer recovery does not restart exhausted inner checks. -Keeping the accepted listener open did not help; a new ICE generation after -underlay recovery delivered three fresh packets in about 63 ms. This distinguishes -check scheduling from permanently retired resources. The isolated authorization -and feedback patches retain this failure. - -The following counterfactuals preserve sockets, keys and NAT mappings. Unless -noted, recovery starts after network restoration; none runs Piik's complete -disconnect-grace, restart and rebuild schedule. - -| Tailcat recovery control | Observation | +Browser edge. The owner stopped integration and field-package work. Reopen only +for a concrete Browser-compatible benefit or an explicitly accepted App-only +scope. The controls established no additional reachable NAT class. + +The isolated experiment remains at `d67f0d6d` on +`spike/magicsock-assessment`, outside Piik's production dependency graph. +`git show d67f0d6d:scripts/tailcat-probe/README.md` retrieves its commands, +fixture boundaries and opt-in dependency patches. The complete experimental +observations remain in `git show 82e3f253:docs/research/nat-traversal.md`. +Its internal package is retired, not a current 2.0 test deliverable. + +#### Reachability And Cost + +The October 5-10, 2026 controls used +[Tailcat](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/README.md): +Magicsock/WireGuard with DERP bootstrap and relay fallback. Its Browser path at +that pin is relay-only. Tailscale's +[Browser P2P proposal](https://github.com/tailscale/tailscale/issues/21027) +uses WebRTC; it is not an implemented Browser Magicsock hole-puncher. + +| Controlled question | Result and boundary | | --- | --- | -| Increase only the fixture's Binding request budget to 64 | Media recovered in about 1.9 seconds inside the fixture's eight-second window. This does not accept a product timeout change. | -| 16 Binding requests, three-second recovery window | Nine-second blackout recovered in about 1.9 seconds across three default-library repetitions; eleven seconds did not recover in budget. Other checked phases reached about 2.9 seconds. The authorization/feedback candidate reproduced both outcomes. | -| One client-side `DiscoPing` before default-Pion restart | Failures remained even after a diagnostic reply; that reply does not prove bidirectional encrypted media readiness. | -| One probe on each side before default-Pion restart | Nine- and eleven-second blackouts recovered in three race-detector repetitions, including both probes and SDP exchange inside three seconds. This requires an isolated, unsupported Server hook. | -| Same bilateral probe begins while the network is still down | After eight seconds of blackout, restoration one or two seconds into the probe still left it waiting until its three-second deadline in repeated race checks. Post-restoration controls continued to pass. | -| Lost bilateral probes, retain the existing media connection | In six race checks, both probes expired while WebRTC was already connected; a subsequent observation received fresh SRTP without new SDP, sockets or credentials. The separate three-second observation is not a product recovery extension. | -| New flow confirms encrypted UDP in both directions before inner ICE | With the listener-state repair and actual Viewer owner, one of four nine-/eleven-second controls recovered; three timed out in the existing three-second preparation, before SDP. The bounded fixture echo changes where waiting occurs, not the underlying recovery limit. | - -These controls limit the bilateral proposal: a -[`DiscoPing` call](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/tailcat.go#L2258) -waits for its own diagnostic reply, not an ongoing connection-readiness event. -Later network restoration cannot guarantee that a lost probe completes. -The earlier passing phase therefore does not establish a universal recovery -barrier, and probe expiry does not establish media failure or justify replacing -healthy media. These controls use fixture SDP exchange; they do not diagnose the -product's Trickle ICE or an existing user's failure. - -Manually phased replacement controls give ICE restart and inner-flow replacement -three seconds each, retaining the tunnel, outer mappings and default Pion checks. -Post-restoration controls pass after eight-, nine- and eleven-second blackouts; -two Tailcat controls starting before restoration also pass two further race -repetitions. These windows are stricter than a replacement receiver's actual -15-second initial-connection deadline, not a reproduction of the Viewer schedule. -A bounded repeated fixture preface prevents a lost first UDP packet from blocking -listener acceptance before ICE. It is not product signaling or media-health proof. - -A separate fixture feeds kernel connection states into the actual -`NativeCapableViewerPeer`. Each rebuild gets a new connection ID, as in the -product, while preserving the parent's recovery budget. Renewing that budget -on replacement hid the nine-second outage: three repetitions recovered only -after a third automatic request, outside the two-request contract. The corrected -owner and fixture retain the count across Native and Browser backend replacement. - -The actual-owner fixture now supports incremental SDP/candidate exchange, -matching Piik's [Trickle ICE generation boundary](https://www.rfc-editor.org/rfc/rfc8838.html#section-13). -With this enabled, ordinary Pion recovers two-, nine- and eleven-second outages -in three runs each. Without it, the same eleven-second control still fails at -the fixture's three-second gathering barrier while the network is unavailable. -That negative control is a fixture limitation, not a product defect. - -In the matched Trickle run with the isolated listener-state repair, Tailcat -recovers two- and seven-second interruptions. Nine- and eleven-second outages -exhaust recovery: the replacement sends eight inner Binding checks, receives -none, and delivers no fresh SRTP during its initial deadline or a separate -three-second observation after exhaustion. Both 16-check cases pass; one -64-check replacement instead fails before SDP while awaiting its inner flow. -Permanent loss exhausts the existing budget and disposal cancels recovery. -No observation grants another attempt or extends a deadline. Native control -and local decoding remain stubs, and events are sampled within 100 ms. These -limits prevent claiming full App integration or diagnosing a field failure; -the composition gap remains an adoption concern. - -Bounded anonymous traces further localize two Tailcat encrypted-echo failures. -The replacement flow is accepted and its client preface read at the server, -but the reverse read expires after three seconds before SDP. The server selects -its direct path near the end of that window; the client has selected its path -about two seconds earlier. No filter-drop event was observed, which is not -packet-delivery proof. Other timing phases recover. This supports a directional -underlay scheduling limit; it does not establish a new authorization defect, -a safe readiness barrier, or a reason to widen product deadlines. - -Varying the existing [Pion Binding-request setting](https://github.com/pion/webrtc/blob/v4.2.19/settingengine.go#L506) -can bridge a discovery delay, but repeated controls also fail before SDP during -new-flow setup. Paired counters separately show the server receiving inner STUN -while the client receives none. More checks cannot repair every setup failure; -neither a new readiness owner nor a longer product deadline is accepted. - -A retained-flow counterfactual reuses Pion's existing -[UDP mux](https://github.com/pion/ice/blob/v4.4.0/udp_mux.go) after closing the old -PeerConnection. It keeps the accepted Tailcat flow and replaces ICE/DTLS/SRTP, -without another listener or preparation exchange. Setup falls from about 913 ms -to 11–20 ms. Nine-/eleven-second recovery passes three of six default-check -cases, four of six with 16 checks and six of six with 64. Removing flow creation -alone therefore does not resolve recovery. Combined listener/revocation/TWCC -patches pass Linux race checks for 64-check recovery, bounded exhaustion, -retirement and sibling audio/video through serial replacements and rollback. -Overlapping connections still use separate flows, and revocation stops both. -This counterfactual does not accept retained flow as a product owner; the fresh -flow alternative below avoids that extra lifetime. - -A preface-free alternative uses Tailcat's existing -[UDP listener](https://github.com/tailscale/tailcat/blob/b4dc28e8aa8936f0a90a41ad8293a64e3d6b645f/listen.go) -to reserve the address before SDP, allowing ICE to supply the first authenticated -datagram. Each replacement owns a fresh flow; no old mux survives it. An -unaccepted write returns a socket error instead of blocking ICE's event loop. -In the combined dependency candidate, 64-check nine-/eleven-second controls -each pass two Linux race repetitions under the actual Viewer owner. Setup takes -12–15 ms; fresh media returns at about 14.2 seconds from blackout onset, within -the unchanged two-request and initial-connection budgets. Permanent loss still -exhausts recovery, and retirement closes pending work. Local race controls cover -mutable deadlines, closing during ICE, peer rejection and continuous sibling -audio/video. Default inner checks still fail in these restart-then-rebuild controls. -This removes the fixture-preface and retained-flow lifetime requirements; it -does not accept a product Binding limit, implement room grants or establish -full App recovery acceptance of this alternative. - -The sender's [restart capability](../standards/routing-transport.md#reconciliation) -changes that schedule. Modeling the current Native sender's first-request -replacement under the same Viewer owner on 2026-10-10, default checks recover both -nine- and eleven-second outages at about 23.3 seconds after two replacements. Their first -replacement still exhausts eight checks; the second starts after its existing -15-second initial deadline. With 64 checks, these controls recover after one -replacement at about 12.2 and 15.3 seconds respectively. Permanent loss exhausts -two replacements at about 38.3 seconds, and disposal cancels pending recovery. -This narrows the earlier failure claim to its tested sender behavior; it is not a universal Native failure -or an accepted product tuning. Native control and decoding remain stubbed in -this owner fixture, distinct from the decoded handoff checks below. - -Separate checks preserve continuous sibling audio/video through a local ICE -restart, failed-flow retirement, both callers' pending cancellation/deadline, -revocation during a Server probe and unknown-peer rejection. Canceling a probe -ends the caller's wait, not upstream packets: existing five-second discovery and -ten-second TSMP timers bound callback retention. These local checks do not prove -impaired-network recovery. - -The opt-in experiment retains the failing single-restart and blocking-probe -assertions. It is not an all-green acceptance gate or a supported adapter. Do not -widen product timeouts or add diagnostic Ping polling to hide these failures. -Room-authorized integration and the complete recovery lifecycle remain unverified. - -A discovery-retained control on 2026-10-08 addresses a limit of closing DERP: -closure also removes its endpoint exchange. The upstream relay's packet-class -counters distinguish discovery from WireGuard traffic. A 500 ms window with -connected WebRTC, at least three fresh tagged SRTP packets and no non-discovery -relay packets proves direct media while keeping endpoint exchange available. -Ordinary NAT passes that proof; blocked UDP correctly remains relay-only. Both -orientations of the hard-NAT pair deliver through DERP without a proved direct -window in 20 seconds. After either gateway mapping changes, media returns in -about six seconds. Default-library repetitions do not prove direct recovery in -20 seconds. The combined isolated candidate proves it once per direction at -about 17 seconds, but neither of two further repetitions per direction does. -This variation does not establish a repeatable candidate advantage or an extra -reachable class. The control makes no explicit rediscovery call; its passing -result validates the measurement and available-media baseline, not universal -direct recovery. It does not run Piik's complete reconnect schedule. - -An actual local-address-change control on 2026-10-09 replaces one endpoint's -IPv4 address, default route and public UDP mapping, allowing upstream network -monitoring to react. Discovery stays available, without an explicit endpoint -update or diagnostic Ping. With Piik's unchanged Viewer recovery owner and -Trickle exchange, three repetitions per direction preserve Tailcat's same media -session: direct delivery returns in 1.25–6.40 seconds with no owner restart. -Both default inner checks and the isolated 64-check candidate pass this control. -The ordinary Pion comparison requests one ICE restart and returns direct media -in 9.14–9.24 seconds. Relay accounting separates first available media from -direct delivery; another two seconds of fresh SRTP after DERP closes confirms it. - -The actual Native Engine's wildcard UDP socket is a separate baseline: both -directions stop delivering after the address change without new SDP, and fresh -Native discovery restores local playback on that same changed network. This -avoids attributing only Pion's default per-interface socket behavior to Piik. -These controls establish a bounded address-handoff benefit, not an additional -initial NAT-reachability class, physical Wi-Fi roaming or complete App/decoder -acceptance. They do not resolve the gateway-only or prolonged-outage controls -above. Supported dependencies, room grants and representative network evidence -remain prerequisites for product adoption. - -The remote-socket overlay extends that comparison through the actual Native -encoded source, sender, receiver and local playback. It leaves product files -untouched and binds the supplied socket to each connection-creation request. -Its fixed 8×8 input exposes forwarding pauses but cannot establish live encoder -recovery; declaring those fixed bytes rate-controlled is only an ablation. -The fixed-frame Viewer-address handoff pauses with both the former and current -socket adapters, including a matched control with isolated loopback playback. -Under that same playback isolation, the real Windows VP8/FFmpeg control passes -with the current adapter in all three checked cases: ordinary unchanged, source -address change and Viewer address change. These controls separate the adapter -change from the fixed-input recovery failure; they do not establish its complete -cause or justify relaxing media acceptance. - -The live control instead uses the Windows VP8 helper, Native output-plan owner -and final FFmpeg decode. The top quarter of the input changes from red to blue -after DERP closure, so buffered pictures cannot satisfy fresh-content acceptance. -About one second of ordinary UDP loss can leave the former pacer sending old -video from a queue of hundreds of packets. DownTrack frame counts precede actual -paced writes and cannot prove delivery. The shared media assessment owns the -[pacing comparison](./node-local-media-design.md#send-queue-recovery); -it must pass constrained-bandwidth recovery as well as this short-loss control. - -Viewer address changes expose a separate confound: same-machine playback can -select a LAN address that the experiment then removes. Restricting only the -fixture receiver to loopback separates that leg from the remote underlay. -With this control and the upstream default send queue, all six cases pass: -unchanged ordinary/Tailcat paths, both address-change directions, and short -ordinary/Tailcat loss. Each retains decoded fresh pictures in every post-DERP -sample. These passes do not accept the default local-playback path or the -complete App. Verify its Browser bridge -and existing recovery owner before adopting a local-address policy. -No additional NAT-reachability class, physical capture result or product -Magicsock integration follows from these controls. - -A same-machine Windows Browser control rejects forcing the product bridge to -loopback. Chrome 154 decodes from a Native sender bound to `127.0.0.1`, but -Firefox 155 fails that connection; the same Firefox fixture decodes with the -current wildcard bind and a non-loopback selected pair. Mozilla also keeps +| Does it reach more NAT classes than fresh ordinary ICE? | In the initial kernel-NAT matrix without a usable gateway mapping, both connected directly through ordinary/ordinary and open/hard gateways in either direction. Neither established direct media through the other tested hard/blocked combinations within the observation windows. DERP delivered otherwise unreachable cases, establishing relay coverage rather than additional P2P coverage. | +| Does network-change handling help? | Some local-address changes recovered on the same session without new SDP. Gateway-only rebinding and longer interruptions had separate discovery/recovery failures. These results do not establish physical roaming, a universal recovery improvement or increased initial reachability. | +| Does it coexist with ordinary ICE? | An isolated optional multi-mux composition retained ordinary media with bootstrap unavailable and used a delayed tunnel candidate when needed. It required dependency patches and readiness/cancellation integration; it is not a supported plug-in. | +| What is the bounded cost? | Matched stripped Windows App builds grew from 44.1 to 59.8 MiB. A local four-Viewer full-App comparison peaked at 81.8/93.2 MiB RSS for ordinary/optional modes; final goroutines returned to the same baseline. Codec and Browser processes were separate. These are not sustained or per-room budgets. | + +Closing DERP removes endpoint exchange as well as relay media. Later controls +therefore retained discovery and counted encrypted relay traffic separately; +a selected direct address alone was insufficient. Tests required fresh media +and either no relayed data or continued delivery after relay closure. The +Native mapping, prediction and dual-stack controls are retained +[below](#gateway-and-address-family-controls). + +#### Integration Boundaries + +The experimental composition exposed costs that a future adoption must address +without another retry owner, timeout extension or silent relay policy change: + +- **Relay policy:** the reviewed upstream API has no direct-only media policy + retaining discovery. An isolated overlay filtered encrypted data in both + directions while allowing discovery/handshakes. This is not an accepted + product mechanism; disabled UDP must not silently become DERP media. +- **Startup and recovery:** outer discovery can outlast the inner ICE check + budget. A diagnostic `DiscoPing` reply did not prove bidirectional media + readiness. Experiments with more checks, extra prefaces and readiness probes + did not establish a universal fix. Initial advertisement also needed an + isolated dependency correction to send direct probes before relying on the + peer's stateful firewall. Ordinary ICE must remain independently usable. +- **Datagram bounds:** Tailcat's reviewed payload ceiling is 1232 bytes. Actual + encrypted TWCC reports reached 1244 bytes, with compound batches larger still; + ordinary Pion delivered them while the tunnel did not. A delayed packet could + grow a report despite a shorter feedback interval. The isolated recorder/writer + split preserves statuses, but is not a supported upstream option. The + [feedback-MTU issue](https://github.com/pion/interceptor/issues/416) and recorded + dependency patches own this finding; reducing RTP size alone is insufficient. +- **Authorization:** the pinned Tailcat/WireGuard composition could retain a + revoked inbound peer or install it from a late authorization result. Scoped + experimental corrections passed revocation, unrelated-peer, sibling-media + and race controls. Piik's production transport does not use these dependencies; + the result is an adoption requirement, not a published Piik defect. +- **Flow ownership:** unrelated listener changes could discard an existing + inbound filter entry. Duplicate client identities could collide at DERP; + close-before-initialization was not terminal. Isolated repairs preserved + filter state, used fresh parent-bound clients and joined canceled creation. + New media flows must retire through their existing connection owner. +- **Optional candidate lifecycle:** stock multi-mux construction and supported + dialing do not independently provide the required delayed socket admission. + A private reservation API plus the existing candidate collector kept ordinary + ICE independent, advertised the tunnel only when ready, fenced cancellation + and delayed end-of-candidates. Its multi-mux also needed a correction for + newly enumerated child addresses. These remain experimental dependency changes. + +#### Scope Of The Retained Evidence + +The isolated overlays exercised real Pion/LiveKit forwarding, H264/VP8 and Opus, +Browser decoding, Native control and room authorization. Bounded rendered checks +covered reconnect, invitation revocation, overlapping parent replacement, +healthy siblings, denial at the room capacity limit and final resource cleanup. +Generated changing pixels distinguished fresh output from retained frames. +The final full-App controls could select the tunnel with ordinary remote +candidates withheld, while ordinary Web edges retained ICE. + +Those checks established local composition and ownership behavior. They did +not establish physical capture, difficult-network benefit, broad mobile support +or sustained field reliability. The experiment's detailed intermediate controls, +failed counterfactuals and package evidence remain in Git rather than this +candidate's work ledger. Independent ordinary-media findings are retained here +and in their existing owners. + +## Ordinary ICE And Media Recovery + +These findings apply without Magicsock. They must remain part of candidate +acceptance independently of the retired experiment. + +### Interface Refresh And Local Browser Bridge + +The local restart control reproduced an address-cache defect: Pion ICE v4.4.0's +[wildcard UDP mux](https://github.com/pion/ice/blob/v4.4.0/udp_mux.go) retained its +own `stdnet.Net` interface snapshot across gatherings. After an interface address +changed, both ICE restart and a new SFU connection sharing that mux could +advertise the removed address. The shared mux now takes a private current +snapshot when gathering. Socket identity and Pion's candidate filters remain; +no monitor or timer is added. The regression covers both restart and a new +connection, and restoring the former constructor fails on the stale candidate. +The [stable Pion update](#stable-pion-interoperability) still needs this adapter. + +Forcing the same-machine Browser bridge to loopback is not a portable solution. +A Windows control decoded through `127.0.0.1` in Chrome 154 but failed in +Firefox 155; Firefox decoded using the current wildcard bind and a non-loopback +pair. Mozilla keeps [ICE loopback disabled by default](https://searchfox.org/firefox-main/source/modules/libpref/init/all.js). -These controls establish a compatibility limit, not its complete Browser cause -or public-Site permission behavior. Keep the current address policy. A terminal -local-bridge failure ends the Host share or delegates Viewer recovery to its -existing connection owner. - -Before local-bridge recovery, complete App UI controls on 2026-10-10 confirmed -this distinction in a disposable Linux network namespace, with automatic -topology optimization disabled and no -product source changes. Chromium 152 retains its loopback playback connection -when the namespace's LAN address changes. Firefox 151 selects that LAN address: -the Viewer resumes decoded pictures through the existing Browser fallback after -the local connection fails, roughly 30 seconds later. A Firefox Host instead -ends sharing on that preview failure, including a still-delivering downstream -connection; the Chromium Host control continues. The Host source is generated -VP8 delivered through the real capture-process interface, not physical capture. -Acceptance checks fresh rendered frames at the end, not accumulated frames from -before the interruption. All controls retire their App, Browsers and interfaces. -These are ordinary-transport controls, not integrated Magicsock or physical -roaming evidence. A faster remote tunnel alone cannot establish complete App -handoff; both the network and Browser playback legs must recover. - -The local restart control exposed a separate address-cache defect: Pion ICE -v4.4.0's [wildcard UDP mux](https://github.com/pion/ice/blob/v4.4.0/udp_mux.go) -retains its own `stdnet.Net` interface snapshot across -gatherings. After an interface address changes, both ICE restart and a new SFU -connection sharing that mux can advertise the removed address. The shared mux -adapter now takes a private current snapshot when gathering; it changes neither -socket identity nor Pion's candidate filters and adds no monitor or timer. -The isolated regression covers restart and a new connection; restoring the -former constructor fails on the stale candidate. Firefox also resumes fresh -rendered frames on the same track after a standard ICE restart, including from -the failed state, while a separately bound loopback sibling keeps receiving. -Replacing only the Native PeerConnection while retaining the Browser connection -instead reported connected without fresh decoded pictures in the same fixture; -it is not the accepted recovery mechanism. - -The 2.0 candidate now uses the existing bridge owner for one standard ICE restart -after a three-second disconnect grace, or immediately after failure. Initial and -restart negotiations use complete local SDP with the request ID fencing the answer; -there is no separate local trickle queue. The negotiation has an eight-second -deadline, including a five-second Native gathering bound. Retirement cancels it; -failure retains the existing Host/Viewer terminal owners. Paused sources need not -produce new frames to prove a transport connection. - -Matched complete App controls retain the same Firefox Host and Viewer media track -through the namespace address change. The Host keeps sharing and decoding, its -downstream Viewer keeps rendering, and the Native Viewer resumes fresh rendered -frames without replacing its route or falling back to Browser reception. -Chromium Host/Viewer controls retain their working loopback leg without a restart. -These controls use ordinary Pion and generated media. They close the reproduced local -bridge gap, not the remaining Magicsock composition, physical roaming or platform -acceptance boundaries. - -The aligned candidate also combines the real Native encoder/forwarder/receiver -with the actual Browser bridge over the experimental Tailcat socket. The fresh -flow adapter now replaces its synthetic preface: SDP starts before peer admission, -and ICE supplies the first datagram. All six FFmpeg-decoded controls pass Linux -race checks. Chromium 152 and Firefox 151 each pass an unchanged ordinary-Pion -control and both Tailcat address-change directions. After DERP closes, new blue -content keeps rendering on the same track. Firefox renegotiates its local leg -once when that endpoint moves; Chromium retains its working loopback leg. Local -HTTP replaces Native control only in this fixture; App authorization, room routing -and prolonged-outage replacement are not exercised. No product timeout, address -policy or inner Binding setting changes for these handoff checks. - -A separate Linux race check now passes those experimental flows through real -Native control commands and the Viewer session's existing receiver owner. -Closing a receiver releases its physical flow; replacement and a healthy sibling -keep receiving fresh RTP after DERP closes, including after late cleanup for the -old connection ID. Stopping reception or closing the control session releases -both remaining flows. Omitting receiver retirement in the test overlay is -detected. Keys are still pre-granted locally: this establishes resource cleanup, -not room authorization, automatic App recovery or Browser decoding in this check. - -The request-scoped adapter removes the former Engine-wide next-socket slot and -temporary bootstrap receiver. A lazy factory reaches the existing physical -socket owner only after input/capacity admission; reuse does not allocate again. -Unsupported-codec failure, independent receiver retirement, control close and -cancellation of a still-opening socket pass under the race detector. The Native -engine, Host, Viewer and control package suites also pass with the overlay. -The complete loopback WebSocket check exposes one additional integration gap: -delayed socket preparation waits for its media owner, while deferred session -close waits for that synchronous command. Client disconnect and server shutdown -both leave it waiting in the unmodified owner. The isolated correction runs -session close on control-context cancellation and joins it before releasing the -capacity slot. Both cases, the direct-owner checks and loopback/control package -suites pass under the race detector. This is an experimental slow-transport -requirement, not an established published failure. The -[authorization boundary](#peer-authorization-boundary) owns the subsequent -Viewer startup and receive-command evidence; no new recovery loop or room -resource registry is added. - -The earlier FFmpeg matrix had one decoded-content failure. Matched synthetic -bitstream/RTP replay identifies a receiver-fixture mismatch: its separate SDK -assembler emits a truncated startup frame and can resume dependent frames after -a sequence gap without their reference. Increasing that builder's window instead -stalls recovery. On identical unchanged and source-handoff recordings, the -product relay-input owner's independent-keyframe boundary produces only -byte-matched source frames, with no decoder errors or corrupted colours. -The fixture now reuses that owner. All six unchanged, address-change and loss -controls pass with actual encoding/decoding and Linux race detection, without -decoder errors or changes to product timeout, adaptation or recovery policy. -The original failed run has no packet recording, so its exact cause remains -unassigned; this correction does not establish universal media continuity. - -Dual-stack namespace controls enabled STUN and alternately blocked IPv4 UDP or -IPv6 forwarding. Both stacks used the working family; Tailcat's direct proof -required fresh bidirectional delivery after DERP closed. One side reporting a -direct address did not establish that both directions were direct. IPv4 TCP -still supplied bootstrap, so this is not an IPv6-only or physical ISP check. -These controls add no demonstrated direct-reachability advantage for Magicsock. - -The actual Native `mediaedge.Engine` Host/Viewer now runs through the same kernel -gateways, including local playback and the unchanged TypeScript survey predictor. -Three Linux race repetitions received ten distinct RTP packets containing the -generated VP8 frame, excluding padding/duplicates. With no gateway service, -mapping enabled did not hold ordinary media behind discovery: delivery took -about 0.8 seconds. Either usable IP family also carried media. Sequential-port -NAT failed without prediction and delivered through a predicted candidate when -enabled, with the changing-port gateway on either endpoint; random-port NAT -remained unreachable in the bounded checks. A real -MiniUPnPd gateway made a hard/hard pair reachable when mapping was enabled. -These are encoded-media checks, not capture, Browser decode or room admission. - -Tailcat's matched controls explicitly link its [optional portmapper feature](https://github.com/tailscale/tailscale/blob/a0e471a35b8f/feature/portmapper/portmapper.go); -clearing its disable flag alone does not register that optional feature. Gateway -mapping then passed three race repetitions after DERP closure. The 20-second -comparison distinguishes fresh direct SRTP from relay delivery and retains the -same gateway filtering. Its relay bootstrap and Piik's local playback differ, -so startup timings are not a first-picture benchmark. - -A 2026-10-08 fairness control found that Tailcat's single bootstrap region leaves -`MappingVariesByDestIP` unknown even with three STUN addresses: upstream -[netcheck](https://github.com/tailscale/tailscale/blob/a0e471a35b8f/net/netcheck/netcheck.go) -cancels the region's other probes after its first reply. A lab-only map projection -gives those real addresses independent probe regions and pins the original relay; -it is not a supported Tailcat option. Twelve Linux race cases verify classification -on the actual media socket, keep media flowing until relay closure at 12 seconds, -and require a fresh payload before the 20-second bound. Random/sequential pairs -remain relay-only in both directions, with or without an absent-gateway mapping -probe; an available gateway still establishes direct media. No extra direct path -over Piik was found. Physical routers and field success rates remain unmeasured. -Product dependencies and routing are unchanged. - -The experiment's README owns commands and fixture boundaries; raw output remains -ignored. Further product work needs a measurable network benefit and accepted -room authorization, resource and relay-operation owners, not more loopback -successes. - -### Peer Authorization Boundary - -The 2026-10-08 three-peer control found that Tailcat forgets a revoked key but -leaves its WireGuard session alive: fresh inbound UDP still arrives after -`DisconnectClient` returns and `PeerKey` no longer recognizes it. A separate -controlled race lets a late positive `AllowClient` result re-admit a revoked -key. Both defects were independently reproduced; Tailcat's checked -[October 7 source](https://github.com/tailscale/tailcat/blob/52fbad323e9d352f1febb623e0975368e9a8cbdd/tailcat.go) -retains both mechanisms. Piik's current transport does not use this dependency. - -An opt-in Tailcat patch syncs the existing engine after releasing the backend -lock and invalidates late results in its pending-hook map. Three Windows -repetitions and three Linux race runs reject both traffic directions and new -admission, preserve two healthy peers' media and permit a later explicit grant. -Removing either correction fails its regression; upstream admission/disconnect -checks pass. This repairs the Tailcat layer only. - -A deeper unit reproduction pauses WireGuard's lazy peer-config lookup, withdraws -authority and calls `RemovePeer`, then releases the old result. `LookupPeer` -still installs an active peer. Both the pinned `0a83b87f7908` and -[October 2 revision](https://github.com/tailscale/wireguard-go/blob/18e99eba5393b8c9ea513ab7a1db1a12625422ca/device/device.go) -reproduce it. An isolated patch on the newer version tracks in-flight config -lookups at the peer table and starts lazy peers under that same lock. Removing -a key retires only its pending calls; lookup replacement and close retire all. -Tokens are reclaimed on completion without accumulating historical keys. -An unrelated-key control rejects a global generation fence: it cancels valid -concurrent admission. Scoped tokens preserve that admission, overlapping grants -and cleanup; removing the retirement check reproduces stale installation. -The full device suite and lock-order analyzer pass, including three Linux race -runs. Combined media and TWCC checks pass on Windows and Linux; the Linux kernel -NAT, dual-stack, rebinding and weak-network controls also pass. Supported dependency -contracts and demonstrated network benefit remain open. The probe retains the -isolated patches; Piik's production transport is unchanged. - -Admission ordering is also an integration constraint. Upstream -[`Client.DialUDPPort`](https://github.com/tailscale/tailcat/blob/52fbad323e9d352f1febb623e0975368e9a8cbdd/tailcat.go) -waits for the server to admit the client's key before returning a socket. -Piik creates the receiver and its socket before returning a media answer. -Authorizing that key only after receiving the answer would therefore introduce -a circular wait. An isolated `PrepareUDPPort` patch reserves the local address -without waiting for admission. Through real Native control, it can return the -answer first and receive fresh RTP if the fixture holds remote ICE candidates -until the grant completes. Both receiver/control retirement paths retain their -cleanup checks; the pre-granted control uses the same client/server orientation. -Without that experimental candidate gate, Native sends five ICE checks before -the roughly one-second grant acknowledgment, reaches eight and receives no RTP -within the unchanged bound. Complete-SDP and early-trickle controls both fail. -The early-trickle counterfactual is opt-in and intentionally fails its media check. - -The simpler pre-ICE control uses supported `DialUDPPort` in both Native -listener/dialer orientations. With keys already admitted, the real Native control -session receives fresh RTP, replaces a flow without interrupting its sibling and -retires all flows on stop or control close. All four Linux race cases and vet -pass with `PrepareUDPPort` removed from the copied dependency, without an explicit -diagnostic ping or an admission-dependent candidate gate. The other isolated -dependency corrections remain. This removes the need for that extra startup API -when key admission precedes media negotiation; it does not implement room-scoped -key exchange or demonstrate higher direct reachability. - -The isolated signaling extension carries bounded listener metadata on the -assigned parent's ordinary SDP offer. SDP/ICE remain the signal variants; -there is no extra key-exchange message or router authorization branch. Actual -room WebSocket checks reject unrelated peers/rooms, retired candidates and -listener metadata on answers or ICE candidates. Ordinary invitation revocation -still closes the authenticated Viewer session. Protocol/signaling race suites -pass. The generated extension adds no server key registry or product wire change. - -A separate Native control check derives admission from its actual owned inbound -flows. Closing one flow retains the shared peer's authorization while another -flow remains; closing the last disconnects that peer. Final revocation and a new -grant serialize without holding the flow-map lock through Tailcat callbacks. -Stop and control-close cases pass under the race detector, including fresh -sibling RTP before preparing a replacement. Counterfactuals that disconnect on -every close or never disconnect fail those checks. This composes physical-flow -cleanup with the Native session; keys in that control remain local fixture inputs. -The pending-flow adapter also rejects a different admitted peer without consuming -the assigned listener. The two-peer control reproduced the former premature -close; the repair permits the expected peer or cancels through the same owner. - -The composed room/Native experiment separates session availability from actual -transport identity. Native control advertises only availability in its ordinary -ready response; authentication and existing assignments project that fact only -to assigned parents. Restart/reconnect can clear it through the same owners. -No persistent device key or room-wide key registry is added. App-launched Viewer -startup uses its existing discovery promise; missing App retains Browser fallback. -Late discovery, unmount, control replacement and stale close remain fenced. - -The parent reserves its listener and sends its bounded metadata with the ordinary -SDP offer. The actual receive command creates or reuses the parent-bound Tailcat -client and emits its public key through a connection-scoped Native event before -dialing. An experimental `native-peer` signal returns that key to the exact -assigned listener; its grant precedes receiver ICE. Parent-side binding checks -the complete listener identity, is immutable after binding, and ends with that -physical flow. Preparing and committed routes use the existing authorizer; -reverse, unrelated, cross-room and retired-connection messages are rejected. -Listener metadata accepts no relay URL or region. Trusted relay configuration -remains local, and diagnostics exclude connection keys. - -On 2026-10-10 the previous session-wide key produced duplicate DERP registrations -when two distinct parent-bound clients overlapped. Upstream -[DERP registration](https://github.com/tailscale/tailscale/blob/main/derp/derpserver/derpserver.go) -selects among duplicate connections; local direct RTP had hidden this conflict. -The corrected adapter uses Tailcat's default fresh identity per client, sharing -that client only across physical flows to the same parent address. Both the -same-parent and independent-tunnel-parent controls now require zero duplicates -and fresh RTP before and after old-flow retirement. These controls use one -assigned room parent with two listener instances; they do not yet prove a full -room-controller handoff between different participant parents. - -Fresh client identity also avoids the measured same-key recreation delay: the -old first WireGuard response went to a cached endpoint and data arrived only on -the next handshake. Last-receiver replacement now meets the original five-second -media bound without resetting the parent, extending timeouts or patching client -discovery lifetime. The superseded dependency patch is removed. Receiver reuse -still requires identical listener metadata; a changed listener replaces the -source/transport through its existing owner. Sibling and ordinary reuse checks -retain their behavior. - -Control close cancels dialing and joins each pending construction before final -client close: Tailcat's close-before-initialization is not terminal. Queued -identity events lose authority with the flow. Native control, loopback and -listener retirement checks cover cancellation and ordinary cleanup; the parent -adapter also retains its already-authorized remote address so a late write fails -normally when the dependency clears its endpoint. - -The isolated Native sender now prepares its listener through the existing -Host/Viewer edge socket factory. Its offer returns metadata, and a strict -connection-scoped control command binds the reply through that same edge. -There is no additional control-level listener registry. The raw transport checks -and Native overlay compile the same admission/socket adapters. Closing an edge -revokes its listener; retiring the source releases its remaining children. -`TestNativeSenderOwnsListener` carries a generated VP8 source through the actual -Native receive/relay commands to two Native children. It checks fresh sibling -RTP after one child retires, idempotent replies and rejection of another listener -or a retired connection. Host/media-edge/Viewer/control race suites pass. This -fixture still pumps events instead of running the Browser route owners; it is -not physical Host capture or complete room-controller acceptance. - -The overlaid Browser NativeViewerPeer forwards the listener and returns scoped -identity events while receive-offer is pending. Wrong share/connection and -post-disposal events are ignored; there is no extra preparation timer or pending -identity registry. The sender reads capability from the current preparation, -assignment or recovery message, without a second capability cache. Existing -Browser owners dispatch the reply to the listener's Native edge. Focused tests -cover stale replies, deep-copied assignment capability and recovery after a -capability change. - -An owned Linux/Chromium 152 run exercises the actual UI and room controller with -a generated 640×360 VP8 canvas source and one outbound copy per participant: -Browser Host → Native Viewer → Native Viewer → Browser Viewer. The inner Native -pair uses Tailcat; the outer Browser links retain ordinary WebRTC. Reconnecting -the second Native Viewer replaces its listener/client without accumulating -resources. After the first Native Viewer leaves, the next Viewer changes to the -Browser Host and both remaining Viewers resume decoded, changing pixels. Removing -the second Native Viewer then restores an ordinary Browser-to-Browser route. -All transport counters are zero at retirement. A second rendered run uses H264 -and three Native Viewers across two App instances. When the middle Native Viewer -leaves, the last Viewer changes to the first Native Viewer, receives a new scoped -connection and resumes decoded, changing pixels; the departed participant retains -no transport resources. The normal App control-session and origin limits remain. -In the mixed Browser/Native chain, revoking a private room's invitation removes -all Viewers and Native transport resources while the Host continues sharing. -The old invitation remains rejected after rotation; a fresh invitation restores -the chain and decoded media without retaining the retired transports. -With copy capacity two, four Native Viewers exercise the overlapping path: -the Browser Host feeds Native A/B, and A initially feeds C/D. Controlled -relay-capacity messages move D from A to B and back through the real controller. -Holding D's first-frame acknowledgement leaves two Native client/flow pairs -alive while its old displayed picture advances; releasing it commits the new -parent and retires only the old pair. C keeps its connection and continues -decoding. Final resource and race checks pass. This exercises transport ownership -under a capacity change, not the quality-degradation trigger or its thresholds. -This is bounded local rendered acceptance, not physical capture, difficult-NAT -reachability or sustained recovery. - -Cold receiver preparation initially took about three seconds in this fixture. -The local DERP region had no STUN service: two preauthorized registration controls -took about four seconds without STUN and 15–16 ms with it. Reusing Piik's STUN -server in the UI fixture reduced cold receive-offer responses to 21–25 ms. Keep -unavailable-STUN controls separate from healthy startup measurements. This finding -changes the test environment, not product timeouts or retry behavior; it does not -predict public-network latency. - -Composition also exposed duplicate address discovery: the shared candidate -collector still sent ordinary STUN probes through the injected virtual socket. -The experimental socket boundary now leaves outer discovery to Tailcat. A local -STUN control fails before this correction and passes after it; ordinary Native -gathering still produces its observed candidate. There is no timeout extension -or additional retry owner. Availability, public keys and handshake replies -remain distinct from decoded-frame readiness and additional field reachability. - -A separate optional-path check uses Pion's public -[`NewMultiUDPMuxDefault`](https://github.com/pion/ice/blob/v4.4.0/udp_mux_multi.go) -to advertise ordinary UDP and one prepared Tailcat flow on the same -PeerConnection. Fresh audio/video passes with either path unavailable, and -normal ICE restart selects the remaining path in both directions. Making the -unused tunnel available or closing it preserves the ordinary session and its -ICE credentials. Windows and Linux race checks pass without an application -path selector or changed ICE budgets. These local tests permit DERP and prepare -the tunnel before ICE; they establish composition, not direct-media policy, -NAT gain or cold-bootstrap independence. - -The default Native overlay replaces its ordinary socket with the tunnel. -`DialUDPPort` waits for remote admission, while Pion's stock multi-mux requires -the sockets' addresses at construction. Simply combining them synchronously -would let unavailable bootstrap delay otherwise usable ordinary ICE. A supported -way to prepare an optional socket independently remains an adoption requirement; -the earlier private `PrepareUDPPort` patch is not a supported upstream API. -Do not replace this gap with two racing PeerConnections, a second recovery -controller or a larger product timeout. - -An isolated Linux race control combines that reservation patch with the same -multi-mux: ordinary SRTP starts while remote admission is denied or bootstrap is -unavailable, and continues through a delayed grant. For a tunnel-only pair, -publishing its candidate only after admission establishes fresh audio/video in -the same ICE generation. Ordinary candidates are available immediately; -end-of-candidates follows the optional producer. Publishing the tunnel candidate -early instead leaves both peers connecting with no media in the matched control. -The candidate ordering uses [Trickle ICE](https://www.rfc-editor.org/rfc/rfc8838.html), -without another PeerConnection, timeout extension or application retry loop. -After ordinary ICE has nominated a pair, a late optional candidate is omitted; -it may participate in a subsequent gathering. This raw fixture permits DERP and -edits its own SDP; it does not establish direct-media policy or greater NAT coverage. - -The separate optional Native overlay now preserves the ordinary socket's STUN -and mapping owner and adds its virtual socket to that same Pion connection. -The receiver's existing candidate collector owns delayed advertisement, EOC and -cancellation; its SDP answer cannot bypass that collector. The existing flow -owner joins admission work on close. Native controls establish ordinary media -before the tunnel grant and with bootstrap unavailable. A tunnel-only control -establishes media after delayed admission, then preserves a sibling and a new -receiver through old cleanup. Candidate events continue during the delay: -bypassing the readiness gate leaves the sender checking with no media. Linux -race checks pass. - -A bounded Chromium 152 UI check also runs the optional composition through a -Browser Host, two Native relays and a Browser Viewer. Changing decoded pixels -advance at every hop, after the second relay's manual reconnect, and after each -Native relay departs. Departure recovery respects the existing signaling grace; -retired Native transport counts reach zero. Host stop projects waiting for a -share, and the race-enabled fixture closes cleanly. This uses a synthetic source -on one local network; ordinary ICE may win. It establishes coexistence and these -recovery paths, not a tunnel-success rate, physical capture or complete App -acceptance. Optional invitation revocation also rejects old links, clears media -resources and permits new Native reception across four grant generations. This -check exposed and now covers the shared Viewer's -[control-retirement gap](./native-client-lifecycle.md#viewer-control-retirement). -A second optional UI check moves one Native Viewer between two Native parents -in both directions through the real room controller. Holding candidate readiness -within the unchanged operation deadline preserves the old video element, stream -and advancing frames while both receiver flows exist. Releasing readiness commits -the new parent and retires only the old flow; a sibling retains its connection -and advancing decoded pictures throughout. Final transport counts reach zero -under the race detector. The fixture disables automatic quality tuning and -changes advertised capacity to isolate overlapping ownership. This local synthetic -check does not measure handoff latency, difficult-network success or tunnel selection. -The optional composition also passes the kernel-NAT real-codec controls with -Chromium 152 and Firefox 151: unchanged paths, either endpoint's address change, -and a short complete interruption. Ordinary Pion supplies unchanged/interruption -controls. New blue content renders on the same track after DERP closes, with -zero relayed encrypted data. Firefox renegotiates its local leg once when its -endpoint moves; Chromium retains that leg. No timeout or address policy changes. -These synthetic-source checks exercise the Native engine and Browser bridge, -not complete App startup, physical roaming or another reachable NAT class. - -A non-race Chromium comparison also measures the isolated Go workers. One -roughly 18-second, 640x360/30 connection used peak RSS of 64/65 MiB for ordinary -sender/receiver workers and 72/72 MiB with the optional tunnel. Live goroutines -rose from 39/65 to 96/122; final cumulative CPU times were 0.48/0.54 seconds and -0.67/0.67 seconds respectively. These are bounded observations, not App budgets: -the codec, Browser and relay run separately, and both controls link the same -binary. Shared listeners and clients also prevent linear extrapolation to a -room. A stripped Windows App build grows from 44.1 to 59.8 MiB with the isolated -composition, using the same embedded UI and module selection. This excludes -capture/tunnel helpers and ZIP compression. Sustained cost remains open. - -The isolated App overlay now installs that constructor in the actual process-level -control factory. Its matched experimental wire identity rejects product builds, -local CLI configuration selects the TLS bootstrap, and DERP media is disabled in -the binary rather than by an optional environment setting. Actual Windows App -and embedded-UI checks retain fresh decoded pixels and manual reconnect through -ordinary ICE when the experimental bootstrap is unavailable. Two independent -control sessions and process retirement also pass a race check. The Linux full-App -fixture also selects Magicsock with ordinary remote candidates withheld, renders -generated Browser pixels through both Native sessions, and restores pictures on -a new selected connection after manual Viewer reconnect. Closing its process-owned -TLS bootstrap still leaves 300 new frames over ten seconds at 640x360/30, with -changed decoded pixels; App retirement passes the race check. This is local -integration evidence, not another reachable NAT class. - -A non-race full-App comparison uses one Native and three plain-Browser Viewers -with a generated 640x360/30 source. Each Viewer receives 300–301 new frames and -changing pixels over its ten-second observation. Ordinary/optional runs peak -at 81.8/93.2 MiB RSS and use 3.76/4.08 seconds of Go CPU during the roughly -forty-second observation sequence. Live goroutines fall from 211/327 to 27 in -both modes after tabs close, then the App exits normally. The optional run -selects Magicsock on the Native pair with ordinary remote candidates withheld; -Web edges retain ordinary ICE. Both measurements include the same owned TLS -bootstrap/STUN fixture and exclude separate Browser/codec processes. This is a -bounded mixed-room cost and retirement check, not endurance, a 20-Viewer budget -or a network-success comparison. - -A bounded full-room run with one Native and nineteen Web Viewers also retains -changing 640x360/30 pictures: each ten-second observation advances 296–301 frames. -The twenty-first code-only attempt receives the existing generic admission -denial, a retained Viewer keeps receiving, and the App exits normally after -all tabs close. The Go process peaks at 99.3 MiB RSS. These Viewers share one -machine; this does not cover twenty Native peers or physical-network endurance. - -A frozen Windows internal package includes the numeric-media candidate, matched -experimental UI, helpers and notices, separate ordinary/experimental launchers -and bilingual paired-test instructions. Final-package checks cover native discovery, -two control sessions, unselected-origin denial, shutdown and Explorer ZIP members -with extracted-file hashes. Its experimental launcher selects the -[upstream public bootstrap](https://github.com/tailscale/tailcat#tailcat) for the -future paired test; local acceptance used owned or unavailable bootstrap only. -Physical capture, public-invitation operation and field benefit remain open. - -This composition also exposes an upstream address-lookup mismatch: Pion's -multi-mux enumerates current child addresses but `GetConn` only consults its -construction-time table. A newly enumerated address therefore has no mux. The -isolated correction resolves uncached addresses against those same child muxes, -without another address cache or socket owner; the unchanged control fails. -A private Linux network namespace also replaces the real interface address -while retaining the sockets and PeerConnections. ICE restart gathers only the -new address and restores fresh phase-tagged audio/video. Removing only the -multi-mux correction loses that candidate in the matched control. The race -check preserves ordinary discovery; it does not exercise physical Wi-Fi handoff -or prove the optional tunnel's recovery benefit. -The copied dependency correction and socket-reservation method remain adoption -requirements, not supported APIs or changes to the published Piik dependency. - -`room-browser.mjs` bundles the real Browser schemas and diagnostics with matching -experimental fields. It accepts the Go fixture's twelve route/restart messages, -rejects malformed capability/listener fields and removes scoped keys from both -console events and exported reports while retaining connection and keyframe -evidence. The Go and Browser optional-map null checks agree. This is a Node-run -producer/consumer check, not Browser UI or complete App transport acceptance. -The product wire and diagnostic implementation remain unchanged. - -These transport checks establish startup-order and ownership constraints, not a -product integration or the cause of unmatched field reports. The earlier candidate gate and unsupported -dependency method are not accepted product mechanisms. -The existing assigned-connection and candidate owners must order admission and -advertisement so the usable tunnel receives live ICE checks; ordinary ICE need -not wait for that optional producer. Accepting arbitrary keys, passing room credentials -into Native control or enlarging its timeout is not an equivalent solution. +These observations establish a compatibility limit, not every Browser permission +or public-Site behavior. Keep the existing address policy. + +Complete App controls on October 10, 2026 reproduced the consequence in a +disposable Linux namespace. Chromium 152 retained its loopback bridge after a +LAN address change. Firefox 151 selected the removed LAN address: its Host +ended an otherwise healthy share, while its Viewer eventually used Browser +fallback. Fresh rendered frames, rather than accumulated frame counts, were +used for acceptance. The source was generated VP8 through the real capture +process boundary, not physical capture. + +The corrected existing bridge owner performs standard ICE restart after a +three-second disconnect grace or immediately on failure. Initial and restart +negotiations use complete SDP and fence answers by request ID. The eight-second +negotiation bound includes up to five seconds of Native gathering; retirement +cancels it. Existing Host/Viewer terminal owners handle failure, and paused +sources need not produce frames to establish the transport connection. +There is no second local candidate queue or recovery controller. + +Matched controls retained the same Firefox Host and Viewer tracks through the +address change. Host sharing and downstream decoding continued; the Native +Viewer recovered without a route replacement or Browser fallback. Chromium +retained its working loopback leg. Replacing only the Native PeerConnection, +while keeping the Browser connection, instead reported connected without fresh +pictures in a counterfactual. The tests therefore support standard ICE restart, +not unilateral reconstruction or a claim of physical roaming coverage. + +### Recovery Ownership And Forwarding + +A real Viewer-owner fixture retained sockets and external conditions through +interruption and replacement. It exposed recovery counts being renewed with a +new Native/Browser backend, permitting an extra request. The corrected budget +belongs to the parent opportunity, as specified by the existing +[recovery owner](../standards/routing-transport.md#reconciliation). +Trickle-exchange controls recovered ordinary Pion after two-, nine- and +eleven-second interruptions in three runs each. A full-gathering fixture failed +while the network was unavailable; that negative control was a fixture limit, +not evidence for extending product deadlines. + +A separate live Windows VP8/Native forwarding control exposed stale decoded +pictures after roughly one second of UDP loss: hundreds of already-encoded +packets remained in the former throttled queue. Forwarder frame counts preceded +actual paced writes and could not prove delivery. With the upstream default send +queue and the local playback leg isolated, all six ordinary/experimental +unchanged, address-change and short-loss controls retained fresh decoded +content after relay closure. The +[send-queue assessment](./node-local-media-design.md#send-queue-recovery) +owns the constrained-bandwidth comparison and adopted correction. This is not +proof that every brief blur or unmatched interruption shares that cause. + +The same full-room work found a denied Viewer retaining its App control slot +across repeated invitation generations. The +[control-retirement assessment](./native-client-lifecycle.md#viewer-control-retirement) +owns that independent repair and its rendered acceptance boundary. + +### Gateway And Address-Family Controls + +The actual Native `mediaedge.Engine` Host/Viewer also ran through Linux kernel +gateways with local playback and the unchanged TypeScript predictor. Three race +repetitions required distinct generated VP8 RTP, excluding padding and duplicate +packets. Without a gateway service, best-effort mapping did not block ordinary +media behind discovery. Either usable IP family carried media when the other +was blocked. Sequential-port NAT failed without prediction and used a predicted +candidate when enabled, with the changing-port gateway on either endpoint; +random-port NAT remained unreachable in the bounded controls. A real MiniUPnPd +gateway made a hard/hard pair reachable with mapping enabled. + +These are encoded-media controls, not physical capture, Browser decode, complete +room admission or a new field-success measurement. The matched Tailcat controls +needed its optional portmapper module and a fair multi-region STUN projection; +they still established no additional direct path. Existing mapping, prediction +and IPv6 should be assessed against the same endpoints and relay policy before +another mechanism is proposed. ## Native Mapping Cache And Retry Lifetimes diff --git a/docs/research/node-local-media-design.md b/docs/research/node-local-media-design.md index 2f8db494f..1ccecb116 100644 --- a/docs/research/node-local-media-design.md +++ b/docs/research/node-local-media-design.md @@ -414,8 +414,8 @@ packets drain in two seconds. The upstream unthrottled writer drains the same input once the writer resumes. A separate real Windows VP8 codec and Native forwarding control reproduces stale decoded pictures after roughly one second of ordinary UDP loss; the packet-count cap alone does not prevent this. -The [Native transport experiment](./nat-traversal.md#integration-cost-and-recovery-controls) -owns the decoded-picture and address-change boundary. +The [ordinary media controls](./nat-traversal.md#recovery-ownership-and-forwarding) +retain the decoded-picture and address-change evidence boundary. Congestion callbacks could also promote the estimator's unmeasured 100 Mbps initial capacity while a probe was pending. Congestion now only reduces the From 680fba2e2dc0ec74a08546098282b15b02c6093d Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:21:23 +0800 Subject: [PATCH 146/262] fix(media): distinguish H264 display areas from decoder storage --- docs/research/native-client-media.md | 24 ++++- native/capture/darwin/README.md | 3 +- native/capture/darwin/main.swift | 69 +++++++++++---- native/capture/windows/h264_decoder.h | 31 ++++++- native/capture/windows/main.cpp | 40 ++++++--- native/capture/windows/output_worker.test.cpp | 83 +++++++++++++++++- native/fixtures/h264-padding/README.md | 22 +++++ native/fixtures/h264-padding/aperture.h264 | Bin 0 -> 25534 bytes 8 files changed, 237 insertions(+), 35 deletions(-) create mode 100644 native/fixtures/h264-padding/README.md create mode 100644 native/fixtures/h264-padding/aperture.h264 diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 614f16f4b..469907aab 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -3,8 +3,8 @@ - Reviewed: 2026-10-06 - Scope: platform capture, shared encoded sources, Pion transport and Browser decode; current behavior belongs to [media quality](../standards/media-quality.md) -- Status: Windows physical native Host and Viewer gates passed; macOS and Linux adapters - compile and package but still require physical media gates +- Status: Windows physical native Host and Viewer gates passed; current macOS + changes await SDK compilation, and macOS/Linux physical media gates remain open ## Windows Audio Exclusion @@ -614,6 +614,26 @@ An isolated Level 5.2 trial was rejected by this machine's hardware encoder at output-type setup; it did not reach the decoder. The candidate therefore keeps Level 5.1 and rejects 4K60 before allocating an encoder. +Decoder format checks must distinguish storage from visible content. On +2026-10-11, the Windows decoder rejected a valid 3840x2160 picture stored in a +3840x2176 surface because its output check applied the visible-pixel bound to +codec padding. Media Foundation's [display aperture](https://learn.microsoft.com/en-us/windows/win32/medfound/picture-aspect-ratio) +and CoreMedia's [clean aperture](https://developer.apple.com/documentation/coremedia/cmvideoformatdescriptiongetcleanaperture(_:originisattopleft:)) +own visible dimensions; both adapters now validate these separately from bounded +coded storage. Windows copies the aperture while retaining the decoder surface, +so downstream outputs receive the visible picture. The native regression checks +right/bottom H.264 padding with real software decoding, visible edge pixels, and +an offset NV12 aperture through the DXGI-buffer path. Oversized coded/reference +storage remains rejected; the macOS SDK boundary above still applies. + +An additional nonzero left/top SPS crop exposed a local Media Foundation software +decoder limit: its raw output omitted right/bottom picture pixels before Piik's +surface copy. FFmpeg decoded the same generated frame correctly. Removing the +input-size hint or changing output-aperture attributes did not restore them. +The passing ordinary alignment-padding controls do not establish arbitrary SPS +crop support on that decoder; no bitstream rewrite or replacement decoder is +introduced for this unreported platform edge case. + Current production Windows code encoded and decoded 30 synthetic frames in each 4K orientation, both with and without a D3D decoder manager. Retaining/uploading the decoded surface also passed. The normal Go/native encoded-input path then diff --git a/native/capture/darwin/README.md b/native/capture/darwin/README.md index f0600fa1c..a2cf08487 100644 --- a/native/capture/darwin/README.md +++ b/native/capture/darwin/README.md @@ -20,7 +20,8 @@ sh native/capture/darwin/build.sh /outside/repository/build ``` `--geometry-self-test` checks content cropping, Retina scaling and rejection of -invalid frame geometry without requiring a hardware encoder. `--self-test` +invalid frame geometry, plus H.264 visible/coded-size admission, without requiring +a hardware encoder. `--self-test` encodes successive landscape, portrait, ultrawide and small 420v buffers through one output worker, requiring the fitted dimensions and a constrained-baseline SPS/PPS/IDR after each size change. Neither needs Screen Recording permission. diff --git a/native/capture/darwin/main.swift b/native/capture/darwin/main.swift index 716fa1724..7ee5e60ce 100644 --- a/native/capture/darwin/main.swift +++ b/native/capture/darwin/main.swift @@ -17,6 +17,8 @@ private let bitrate = 3_000_000 private let maxPayloadBytes = 4_194_304 private let maxVideoDimension = 3840 private let maxVideoPixels = 3840 * 2160 +private let maxCodedVideoDimension = 4096 +private let maxCodedVideoPixels = 36_864 * 16 * 16 private let supportedH264Levels: Set = [ 0x1e, 0x1f, 0x20, 0x28, 0x29, 0x2a, 0x32, 0x33, ] @@ -1168,6 +1170,22 @@ private func geometrySelfTest() throws { guard VideoImage(buffer, attachments: outside) == nil else { throw CaptureFailure(description: "out-of-bounds capture content was accepted") } + // Generated Constrained Baseline: 3856x2176 storage, 3840x2160 visible. + // Exercise CoreMedia's SPS interpretation without allocating a decoder. + let sps = Data([0x67, 0x42, 0xc0, 0x33, 0xd9, 0x00, 0x3c, 0x40, 0x44, + 0x78, 0x98, 0x9c, 0x04, 0x40, 0x00, 0x00, 0x03, 0x00, + 0x40, 0x00, 0x00, 0x0c, 0x23, 0xc6, 0x0c, 0x92]) + let pps = Data([0x68, 0xcb, 0x83, 0xcb, 0x20]) + let format = try h264InputFormat(sps: sps, pps: pps) + guard CMVideoFormatDescriptionGetCleanAperture(format, originIsAtTopLeft: true) == + CGRect(x: 0, y: 0, width: 3840, height: 2160) else { + throw CaptureFailure(description: "H.264 visible aperture self-test failed") + } + let oversized = Data([0x67, 0x42, 0xc0, 0x33, 0xda, 0x00, 0xf0, 0x01, + 0xe1, 0xf8, 0x03, 0x49, 0x40]) + guard (try? h264InputFormat(sps: oversized, pps: pps)) == nil else { + throw CaptureFailure(description: "H.264 crop concealed oversized storage") + } } private final class AudioCaptureOutput: NSObject, SCStreamOutput, SCStreamDelegate { @@ -1625,6 +1643,39 @@ private func readInput(done: StopSignal, outputs: [OutputProfile] = [], } } +private func h264InputFormat(sps: Data, pps: Data) throws -> CMVideoFormatDescription { + var format: CMFormatDescription? + let status = sps.withUnsafeBytes { first in + pps.withUnsafeBytes { second in + let pointers = [first.bindMemory(to: UInt8.self).baseAddress!, + second.bindMemory(to: UInt8.self).baseAddress!] + let sizes = [sps.count, pps.count] + return CMVideoFormatDescriptionCreateFromH264ParameterSets( + allocator: kCFAllocatorDefault, parameterSetCount: 2, + parameterSetPointers: pointers, parameterSetSizes: sizes, + nalUnitHeaderLength: 4, formatDescriptionOut: &format) + } + } + try require(status, "decode-h264-format") + guard let format else { throw CaptureFailure(description: "missing input format") } + let dimensions = CMVideoFormatDescriptionGetDimensions(format) + let width = Int(dimensions.width), height = Int(dimensions.height) + guard width >= 2, width <= maxCodedVideoDimension, width % 2 == 0, + height >= 2, height <= maxCodedVideoDimension, height % 2 == 0, + width * height <= maxCodedVideoPixels else { + throw CaptureFailure(description: "decoded input storage exceeds its bound") + } + // Encoded dimensions include padding; the clean aperture owns visible size. + let area = CMVideoFormatDescriptionGetCleanAperture(format, originIsAtTopLeft: true) + let bounds = CGRect(x: 0, y: 0, width: width, height: height) + guard area.minX.isFinite, area.minY.isFinite, area.width.isFinite, area.height.isFinite, + area == area.integral, bounds.contains(area), + validVideoSize(Int(area.width), Int(area.height)) else { + throw CaptureFailure(description: "decoded input aperture exceeds its picture bound") + } + return format +} + private final class H264InputDecoder { private let encoders: EncoderGroup private let lock = NSLock() @@ -1692,23 +1743,7 @@ private final class H264InputDecoder { guard recovery, !nextSPS.isEmpty, !nextPPS.isEmpty else { throw CaptureFailure(description: "new H.264 input needs SPS, PPS and IDR") } - var nextFormat: CMFormatDescription? - let status = nextSPS.withUnsafeBytes { first in - nextPPS.withUnsafeBytes { second in - let pointers = [first.bindMemory(to: UInt8.self).baseAddress!, second.bindMemory(to: UInt8.self).baseAddress!] - let sizes = [nextSPS.count, nextPPS.count] - return CMVideoFormatDescriptionCreateFromH264ParameterSets( - allocator: kCFAllocatorDefault, parameterSetCount: 2, - parameterSetPointers: pointers, parameterSetSizes: sizes, - nalUnitHeaderLength: 4, formatDescriptionOut: &nextFormat) - } - } - try require(status, "decode-h264-format") - guard let nextFormat else { throw CaptureFailure(description: "missing input format") } - let dimensions = CMVideoFormatDescriptionGetDimensions(nextFormat) - guard validVideoSize(Int(dimensions.width), Int(dimensions.height)) else { - throw CaptureFailure(description: "decoded input exceeds its dimension bound") - } + let nextFormat = try h264InputFormat(sps: nextSPS, pps: nextPPS) if let session, !VTDecompressionSessionCanAcceptFormatDescription(session, formatDescription: nextFormat) { VTDecompressionSessionInvalidate(session) self.session = nil diff --git a/native/capture/windows/h264_decoder.h b/native/capture/windows/h264_decoder.h index af776bc95..76cced74e 100644 --- a/native/capture/windows/h264_decoder.h +++ b/native/capture/windows/h264_decoder.h @@ -63,6 +63,8 @@ class H264Decoder final { return windows::DecodedVideoColor(type.Get()); } + RECT DisplayArea() const { return display_area_; } + void Decode(const InputEnvelope& input, const Consumer& consume) { if (input.kind != 2 || input.data.empty() || input.data.size() > 4 * 1024 * 1024) throw std::runtime_error("Invalid H264 decoder input"); @@ -117,9 +119,31 @@ class H264Decoder final { if (subtype != MFVideoFormat_NV12) continue; UINT32 width = 0, height = 0; Check(MFGetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, &width, &height), "output-size"); - if (!ValidVideoSize(width, height)) { - throw std::runtime_error("H264 decoder output dimensions exceed the source bound"); + if (!ValidCodedVideoSize(width, height)) { + throw std::runtime_error("H264 decoder output storage exceeds its bound"); + } + // MF_MT_FRAME_SIZE includes codec padding. The display aperture owns + // visible pixels, independently of the bounded decoder allocation. + MFVideoArea area{}; + UINT32 area_bytes = 0; + HRESULT aperture = type->GetBlob(MF_MT_MINIMUM_DISPLAY_APERTURE, + reinterpret_cast(&area), sizeof(area), &area_bytes); + if (aperture == MF_E_ATTRIBUTENOTFOUND) + aperture = type->GetBlob(MF_MT_GEOMETRIC_APERTURE, + reinterpret_cast(&area), sizeof(area), &area_bytes); + if (aperture == MF_E_ATTRIBUTENOTFOUND) { + area = {{0, 0}, {0, 0}, {static_cast(width), static_cast(height)}}; + } else { + Check(aperture, "display-aperture"); + if (area_bytes != sizeof(area)) throw std::runtime_error("H264 display aperture is incomplete"); } + if (area.OffsetX.value < 0 || area.OffsetY.value < 0 || + area.OffsetX.fract || area.OffsetY.fract || + (area.OffsetX.value & 1) || (area.OffsetY.value & 1) || + !ValidVideoSize(area.Area.cx, area.Area.cy) || + area.OffsetX.value + area.Area.cx > static_cast(width) || + area.OffsetY.value + area.Area.cy > static_cast(height)) + throw std::runtime_error("H264 display aperture exceeds its picture bound"); Check(decoder_->SetOutputType(0, type.Get(), 0), "set-output"); UINT32 stride = 0; if (FAILED(type->GetUINT32(MF_MT_DEFAULT_STRIDE, &stride))) { @@ -131,6 +155,8 @@ class H264Decoder final { width_ = width; height_ = height; stride_ = static_cast(stride); + display_area_ = {area.OffsetX.value, area.OffsetY.value, + area.OffsetX.value + area.Area.cx, area.OffsetY.value + area.Area.cy}; return; } throw std::runtime_error("H264 decoder has no NV12 output"); @@ -174,6 +200,7 @@ class H264Decoder final { UINT32 width_ = 0; UINT32 height_ = 0; LONG stride_ = 0; + RECT display_area_{}; bool configured_ = false; }; diff --git a/native/capture/windows/main.cpp b/native/capture/windows/main.cpp index 5ff8a09d1..9f1a5eae1 100644 --- a/native/capture/windows/main.cpp +++ b/native/capture/windows/main.cpp @@ -1662,28 +1662,38 @@ ComPtr UploadDecodedNV12(const DeviceContext& device, } ComPtr OwnDecodedTexture(const DeviceContext& device, IMFSample* sample, - UINT32 width, UINT32 height, LONG stride) { - ComPtr buffer; - Check(sample->GetBufferByIndex(0, &buffer), "decoded-buffer"); - ComPtr dxgi; - if (SUCCEEDED(buffer.As(&dxgi))) { - ComPtr original; - UINT subresource = 0; - Check(dxgi->GetResource(IID_PPV_ARGS(&original)), "decoded-dxgi-resource"); - Check(dxgi->GetSubresourceIndex(&subresource), "decoded-dxgi-subresource"); + UINT32 width, UINT32 height, LONG stride, + const RECT& display_area) { + // Copy the visible aperture while taking ownership of the decoder's storage. + // Downstream converters receive ordinary unpadded NV12, on either decoder path. + auto copy_visible = [&](ID3D11Texture2D* original, UINT subresource) { D3D11_TEXTURE2D_DESC description{}; original->GetDesc(&description); if (description.Format != DXGI_FORMAT_NV12 || description.Width < width || description.Height < height || !piik::capture::ValidCodedVideoSize(description.Width, description.Height)) Fail("decoded-texture-size", "decoder texture exceeds source bounds"); + description.Width = display_area.right - display_area.left; + description.Height = display_area.bottom - display_area.top; description.ArraySize = description.MipLevels = 1; description.Usage = D3D11_USAGE_DEFAULT; description.CPUAccessFlags = description.MiscFlags = 0; description.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET; ComPtr owned; Check(device.device->CreateTexture2D(&description, nullptr, &owned), "decoded-owned-texture"); - device.context->CopySubresourceRegion(owned.Get(), 0, 0, 0, 0, original.Get(), subresource, nullptr); + const D3D11_BOX source{static_cast(display_area.left), static_cast(display_area.top), 0, + static_cast(display_area.right), static_cast(display_area.bottom), 1}; + device.context->CopySubresourceRegion(owned.Get(), 0, 0, 0, 0, original, subresource, &source); return owned; + }; + ComPtr buffer; + Check(sample->GetBufferByIndex(0, &buffer), "decoded-buffer"); + ComPtr dxgi; + if (SUCCEEDED(buffer.As(&dxgi))) { + ComPtr original; + UINT subresource = 0; + Check(dxgi->GetResource(IID_PPV_ARGS(&original)), "decoded-dxgi-resource"); + Check(dxgi->GetSubresourceIndex(&subresource), "decoded-dxgi-subresource"); + return copy_visible(original.Get(), subresource); } const DWORD size = width * height * 3 / 2; std::vector pixels(size); @@ -1707,7 +1717,11 @@ ComPtr OwnDecodedTexture(const DeviceContext& device, IMFSample Check(buffer->Unlock(), "decoded-memory-unlock"); if (!valid) Fail("decoded-memory-size", "NV12 output buffer is incomplete"); } - return UploadDecodedNV12(device, pixels.data(), width, height); + auto uploaded = UploadDecodedNV12(device, pixels.data(), width, height); + if (display_area.left == 0 && display_area.top == 0 && + display_area.right == static_cast(width) && display_area.bottom == static_cast(height)) + return uploaded; + return copy_visible(uploaded.Get(), 0); } void RunEncodedVideo(ProductArguments arguments) { @@ -1778,7 +1792,9 @@ void RunEncodedVideo(ProductArguments arguments) { Check(sample->GetSampleTime(×tamp), "decoded-timestamp"); Check(sample->GetSampleDuration(&duration), "decoded-duration"); if (timestamp < 0 || duration <= 0) Fail("decoded-timing", "decoder did not preserve source timing"); - submit(OwnDecodedTexture(device, sample, width, height, stride), width, height, + const auto area = h264->DisplayArea(); + submit(OwnDecodedTexture(device, sample, width, height, stride, area), + area.right - area.left, area.bottom - area.top, static_cast(timestamp), static_cast(duration), h264->ColorSpace()); }); } else { diff --git a/native/capture/windows/output_worker.test.cpp b/native/capture/windows/output_worker.test.cpp index bd5aeb170..2110b2ded 100644 --- a/native/capture/windows/output_worker.test.cpp +++ b/native/capture/windows/output_worker.test.cpp @@ -1,11 +1,14 @@ // Exercise the production worker with a synthetic codec at its existing link -// boundary. WARP supplies only the device-status API; no screen or audio is used. +// boundary. WARP also checks decoded-surface copies; no screen or audio is used. #define wmain capture_main #include "main.cpp" #undef wmain #include +#include +#include #include +#include namespace piik::capture::windows { class AdaptiveEncoder::Impl final { @@ -309,6 +312,83 @@ void CheckSurfaceSample(ID3D11Device* device) { assert(time == 1'000'000 && duration == 333'333); } +void CheckDecodedDisplayArea(ID3D11Device* device) { + Runtime runtime; + DeviceContext context; + context.device = device; + device->GetImmediateContext(&context.context); + auto check_red = [&](ID3D11Texture2D* texture) { + D3D11_TEXTURE2D_DESC description{}; + texture->GetDesc(&description); + assert(description.Width == 3840 && description.Height == 2160 && + description.Format == DXGI_FORMAT_NV12); + description.Usage = D3D11_USAGE_STAGING; + description.BindFlags = 0; + description.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + ComPtr staging; + Check(device->CreateTexture2D(&description, nullptr, &staging), "fixture-readback"); + context.context->CopyResource(staging.Get(), texture); + D3D11_MAPPED_SUBRESOURCE mapped{}; + Check(context.context->Map(staging.Get(), 0, D3D11_MAP_READ, 0, &mapped), "fixture-map"); + const auto* pixels = static_cast(mapped.pData); + for (UINT y : {0u, 1080u, 2159u}) { + for (UINT x : {0u, 1920u, 3839u}) { + const size_t uv = static_cast(2160 + y / 2) * mapped.RowPitch + (x & ~1u); + if (pixels[static_cast(y) * mapped.RowPitch + x] <= 60 || + pixels[uv] >= 130 || pixels[uv + 1] <= 180) + std::cerr << "aperture pixel " << x << ',' << y << " YUV=" + << unsigned(pixels[static_cast(y) * mapped.RowPitch + x]) << ',' + << unsigned(pixels[uv]) << ',' << unsigned(pixels[uv + 1]) << '\n'; + assert(pixels[static_cast(y) * mapped.RowPitch + x] > 60); + assert(pixels[uv] < 130 && pixels[uv + 1] > 180); + } + } + context.context->Unmap(staging.Get(), 0); + }; + + const auto path = std::filesystem::path(__FILE__).parent_path().parent_path().parent_path() / + "fixtures" / "h264-padding" / "aperture.h264"; + std::ifstream file(path, std::ios::binary); + assert(file); + piik::capture::InputEnvelope input; + input.kind = 2; + input.recovery = true; + input.width = 3840; + input.height = 2160; + input.duration = 10'000'000 / 24; + input.data = std::vector(std::istreambuf_iterator(file), {}); + unsigned decoded = 0; + piik::capture::H264Decoder decoder(nullptr); + for (unsigned index = 0; index < 4; ++index) { + input.timestamp = (index + 1) * input.duration; + decoder.Decode(input, [&](IMFSample* sample, UINT32 width, UINT32 height, LONG stride) { + const auto area = decoder.DisplayArea(); + assert(width == 3856 && height == 2176 && area.left == 0 && area.top == 0 && + area.right == 3840 && area.bottom == 2160); + check_red(OwnDecodedTexture(context, sample, width, height, stride, area).Get()); + ++decoded; + }); + } + assert(decoded > 0); + + // Exercise the DXGI-buffer path with known NV12 padding as well. + constexpr UINT width = 3872, height = 2176; + std::vector pixels(width * height * 3 / 2); + for (UINT y = 0; y < height; ++y) { + for (UINT x = 0; x < width; ++x) { + pixels[y * width + x] = x >= 16 && x < 3856 && y >= 8 && y < 2168 ? 81 : 41; + if (y < height / 2) { + const bool red = x >= 16 && x < 3856 && y >= 4 && y < 1084; + pixels[(height + y) * width + x] = red ? (x % 2 ? 240 : 90) : (x % 2 ? 110 : 240); + } + } + } + auto texture = UploadDecodedNV12(context, pixels.data(), width, height); + auto sample = CreateSurfaceSample(texture.Get(), 1'000'000, 416'666); + check_red(OwnDecodedTexture(context, sample.Get(), width, height, width, + RECT{16, 8, 3856, 2168}).Get()); +} + int main() { VideoProfile profile{2560, 1440, 60, 12'000'000}; ValidateVideoProfile(profile); @@ -345,6 +425,7 @@ int main() { Check(D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_WARP, nullptr, 0, nullptr, 0, D3D11_SDK_VERSION, &device, nullptr, nullptr), "fixture-warp"); CheckSurfaceSample(device.Get()); + CheckDecodedDisplayArea(device.Get()); CheckAutoCadenceProbe(device.Get()); CheckWorkerFailureGeneration(device.Get(), false); CheckWorkerFailureGeneration(device.Get(), true); diff --git a/native/fixtures/h264-padding/README.md b/native/fixtures/h264-padding/README.md new file mode 100644 index 000000000..7aa24fe31 --- /dev/null +++ b/native/fixtures/h264-padding/README.md @@ -0,0 +1,22 @@ +# H.264 display aperture fixture + +`aperture.h264` is one generated Constrained Baseline frame with 3856×2176 +encoded storage and a 3840×2160 visible aperture at (0, 0). The visible picture +is red; the right/bottom padding is blue. It contains no screen recording or user data. + +The Windows output-worker test decodes it through Media Foundation and verifies +the dimensions and NV12 pixels handed to downstream encoders. The macOS geometry +self-test uses its SPS/PPS to check CoreMedia's format interpretation without +allocating a hardware decoder. Both run through `npm run check:native` on their +respective platforms. + +To regenerate with FFmpeg and libx264: + +```sh +ffmpeg -f lavfi -i color=c=blue:s=3856x2176:r=24 \ + -vf drawbox=x=0:y=0:w=3840:h=2160:color=red:t=fill \ + -frames:v 1 -c:v libx264 -profile:v baseline -level:v 5.1 \ + -preset medium -tune zerolatency -threads 2 \ + -bsf:v h264_metadata=crop_right=16:crop_bottom=16 \ + aperture.h264 +``` diff --git a/native/fixtures/h264-padding/aperture.h264 b/native/fixtures/h264-padding/aperture.h264 new file mode 100644 index 0000000000000000000000000000000000000000..e76960b616021b98923846fd1ff47ec6c723f3b9 GIT binary patch literal 25534 zcmeI3y^GX97>7qaE!4(N#9}~;L%iK2yRPTq!0Gu7qLrP7+3f5kY`$(X>+M<{XdxmN z7Ghx`+Su6$R*HrAfq#G=SXc=nC>Da9zBv@JnMwwZ`-LT$%sVp~o;>ecZ5U(g<_|w> zEwh8OM@F~rOw2O2fzfH}t`}QxGK$7;F2CV)-L&!N&kOI5etPn8`NZQ}d*AWpU0=SA zf=-)H@u<*>`_$Ae-Kh(-UTZ4wf);hg z#*{P7=HlXF<6_cNMJn?~q34_R>W$cBDUB&glN5P1!z0<1k?^=yebM4Q)lG|NAp99` z#^W-V>9|tD8`1s9_kErz5vviehFxmy=Vdh(etiygPxuXwCeev!$w>9Wdc}H1sAWE{ zgx}#&tcy&BG}Y%ut29k2YE6%(dy%0nL<)~;3n2IqY?p6i((*Sx(WaGRcR8@UjB6We5Iovo$Hl{MM)-2y`dyGN~e-W zQ-3!dYB>(0A}i&})`*qSGRbKz8mMJGrmtnDDrzmNWIi?}wFYWTVG->6(+smD7eSj> zk;+vxG@|X*Ptvl+qg55tBpr!J_j%SOCIp+)jR;x|k1y2E7ha=7&9Yu-BL5<;DJWe!M#G!oI310W6376&XL)M;1d zfHX)W5nVX|(jaYdz!E~8c4ZDogESJ+l>;CR(iR6SA=GJC=9nM#CkJdjo4h)E>f83a zY)^A1o46k>v8RvjtgYkNB-=dslC692>H`brR^LYl!?$O!X-9bj(w6G|Uqc}MUvXM0 z1W1E4^l;?>NQ1P+0ZRyV+LbvV4bn(NR}O$QNLw7RgixnlnFG=wjYM?i07!$h#Q{qQ zb=s9VAPv$;L{|=gG)P+ Date: Sun, 11 Oct 2026 00:35:07 +0800 Subject: [PATCH 147/262] docs(guide): explain numeric media controls --- docs/guide/getting-started.md | 7 +++++++ docs/guide/getting-started.zh-CN.md | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 22c8f2665..4cf6837dc 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -149,6 +149,13 @@ sharing to allow that fallback. The project's online site uses P2P only. ## Picture and sound settings Open **Sharing settings** below the picture to adjust picture and audio quality. + +- Choose a picture preset, or enter width and height under **Resolution ceiling**. + Piik preserves the source's proportions within that ceiling. Selecting 4K sets + the frame-rate ceiling to at most 30 fps. +- Choose an audio preset, or enter an **Audio bitrate limit** from 6 to 320 kbps. + Picture and sound still adapt to the source, device and available bandwidth. + Piik remembers your applied settings for the next visit. Browser-only use remembers them for the current site; opening Piik through an updated App also keeps them when your public-invite address changes. diff --git a/docs/guide/getting-started.zh-CN.md b/docs/guide/getting-started.zh-CN.md index 4f6ae532b..629c3471b 100644 --- a/docs/guide/getting-started.zh-CN.md +++ b/docs/guide/getting-started.zh-CN.md @@ -132,6 +132,12 @@ Linux 原生采集还需要[系统组件](../../native/capture/linux/README.md) ## 画面与声音设置 在画面下方打开 **分享设置**,调整画质和音频质量。 + +- 可选画质预设,也可在 **分辨率上限** 中填写宽和高。画面在上限内保留来源比例; + 选择 4K 时,帧率上限最高为 30 帧。 +- 音频保留三档预设,也可填写 6–320 kbps 的 **音频码率上限**。 + 实际画质和声音仍会根据来源、设备与可用带宽调整。 + Piik 会记住已应用的设置,供下次使用。仅用浏览器时,设置保存在当前站点; 通过新版 App 打开时,更换公网邀请网址也能保留设置。 From c38c892955cde572887df3910c904d149ab36ffb Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:54:58 +0800 Subject: [PATCH 148/262] docs(release): explain major upgrade steps at download entry --- docs/guide/getting-started.md | 16 ++++++++++++++++ docs/guide/getting-started.zh-CN.md | 13 +++++++++++++ site/index.html | 4 ++-- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 4cf6837dc..9f8d3d559 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -121,6 +121,22 @@ participants. This mode has no media-server fallback, and the temporary address has no uptime guarantee. All modes need a working UDP media path. The [App guide](../../cmd/piik-app/README.md#modes) explains the connection setup. +### Upgrading + +**Piik 2.0 and 1.x cannot be used together.** The App and the site it connects to +need the same major version. Read the release notes on +[GitHub](https://github.com/TNTcraftHIM/Piik/releases/latest) or +[Gitee](https://gitee.com/TNTcraftHIM/Piik/releases) before upgrading. + +- **Local room or Public invite:** stop sharing, close the old App and extract + the complete new package. Start the new App and send a fresh invitation. +- **An existing site:** the administrator updates the Server first, following + the [update guide](../operations/service-management.md). Then update the Apps + connecting to that site and refresh all sharing and viewing pages. + +Restarting an App or Server interrupts active shares. This upgrade does not +change the room database schema. + ## Share from an existing site You need an existing HTTPS site running Piik Server, provided by a friend or diff --git a/docs/guide/getting-started.zh-CN.md b/docs/guide/getting-started.zh-CN.md index 629c3471b..82b0d44cc 100644 --- a/docs/guide/getting-started.zh-CN.md +++ b/docs/guide/getting-started.zh-CN.md @@ -108,6 +108,19 @@ Linux 原生采集还需要[系统组件](../../native/capture/linux/README.md) 临时地址也不保证持续可用。所有模式都需要可用的 UDP 媒体路径。 具体连接方式见 [App 指南](../../cmd/piik-app/README.zh-CN.md#运行模式)。 +### 版本升级 + +**Piik 2.0 与 1.x 不能混用。** App 与连接的站点需要使用同一大版本。 +升级前,请先查看 [GitHub](https://github.com/TNTcraftHIM/Piik/releases/latest) 或 +[Gitee](https://gitee.com/TNTcraftHIM/Piik/releases) 上的版本说明。 + +- **本地房间或公网邀请:** 停止分享并关闭旧 App,完整解压新版程序包, + 启动后重新发送邀请链接。 +- **已有站点:** 管理员先按[更新指南](../operations/service-management.zh-CN.md)升级 Server, + 然后更新连接该站点的 App,并刷新所有分享与观看页面。 + +重启 App 或 Server 会中断正在进行的分享。本次升级不变更房间数据库结构。 + ## 使用已有站点分享 需要先有一个运行 Piik Server 的 HTTPS 站点,可以使用朋友或管理员提供的地址。 diff --git a/site/index.html b/site/index.html index ab0b6f20c..bcbf213ab 100644 --- a/site/index.html +++ b/site/index.html @@ -736,8 +736,8 @@

Get Piik App.

- Choose your system, then download and extract. - 选择对应系统,下载后解压。 + Choose your system, then download and extract.
For a major upgrade, check compatibility before updating.
+ 选择对应系统,下载后解压。
跨大版本升级前,请先查看兼容性说明。

From fac7444e4708c14539b426a3a451f796a790f9bf Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:01:44 +0800 Subject: [PATCH 149/262] test(media): correct macOS clean-aperture coordinates Core Video returns clean rectangles in lower-left coordinates, while aperture offsets move downward for positive values. Correct the synthetic crop expectation without changing capture behavior. Mac SDK execution remains pending. --- native/capture/darwin/main.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/native/capture/darwin/main.swift b/native/capture/darwin/main.swift index 7ee5e60ce..65f036c78 100644 --- a/native/capture/darwin/main.swift +++ b/native/capture/darwin/main.swift @@ -1145,10 +1145,12 @@ private func geometrySelfTest() throws { .contentRect: rect.dictionaryRepresentation, .scaleFactor: CGFloat(2), .contentScale: CGFloat(1), ] + // Core Video returns a lower-left origin; the surface crop starts 80 px + // from the top, so its lower edge is 720 - 80 - 600 = 40 px from the bottom. guard let image = VideoImage(buffer, attachments: attachments), try image.fitted(to: ceiling).width == 400, try image.fitted(to: ceiling).height == 600, - CVImageBufferGetCleanRect(buffer) == CGRect(x: 100, y: 80, width: 400, height: 600) else { + CVImageBufferGetCleanRect(buffer) == CGRect(x: 100, y: 40, width: 400, height: 600) else { throw CaptureFailure(description: "capture content geometry self-test failed") } var upscaled = attachments From c93c84f9ef4ecce90a24a2bbe39cf65ab83392d5 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:19:43 +0800 Subject: [PATCH 150/262] docs(preview): clarify remaining candidate acceptance --- docs/status.md | 6 +++--- docs/todo.md | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/status.md b/docs/status.md index 5f550cea1..fe29483df 100644 --- a/docs/status.md +++ b/docs/status.md @@ -1,6 +1,6 @@ # Current Status -Last updated: 2026-10-10 +Last updated: 2026-10-11 This is the compact execution/deployment index. Product modules own behavior, [verification status](./verification-status.md) owns unresolved physical limits, @@ -71,8 +71,8 @@ Custom width/height controls apply combined size and frame-rate limits, admittin The [media assessment](./research/native-client-media.md#media-capability-extension-assessment) owns local codec, derived-output, SFU, Browser/Native fanout and live-control evidence. The macOS geometry implementation still needs SDK compilation and -hardware verification; cross-browser/device and route-change acceptance, and -additional codecs remain open. +hardware verification; cross-browser/device and route-change acceptance remain +open. Additional codecs are not part of this candidate. Magicsock adoption is no-go for this candidate: its App-to-App augmentation does not improve the core App-to-Browser use case. The [assessment](./research/nat-traversal.md#integration-cost-and-recovery-controls) diff --git a/docs/todo.md b/docs/todo.md index 59d5b6d57..601f8a052 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -1,6 +1,6 @@ # Current TODO Ledger -Last reviewed: 2026-10-10 +Last reviewed: 2026-10-11 The current 2.0 preparation includes independent investigation across this ledger. Explicit owner holds remain in force; an investigation does not itself @@ -58,8 +58,8 @@ Implementation needs a concrete useful result within the product model. separate UI presets, numeric requests, bounded implementation capabilities and actual output. Numeric audio and video controls are implemented; the assessment owns local capture, codec, derived-output, SFU and bounded mixed-route evidence. - Extend Browser interoperability and mixed-topology checks to limited receivers - and physical camera orientation behavior. + Remaining receiver acceptance needs physical limited decoders and camera + orientation behavior, beyond the bounded SDP-limit and mixed-route controls. Browser display capture, Browser relay and Browser-to-Native fanout now have bounded high-resolution evidence. macOS geometry SDK/hardware checks and physical 4K/game capture remain open. Preserve safety limits and @@ -68,12 +68,12 @@ Implementation needs a concrete useful result within the product model. retain explicit receive-limit enforcement and the current 320 kbps sender ceiling. Physical listening, mobile and field-network acceptance remain open; research owns the bounded real-Opus impairment and video-recovery control. - New codecs need complete - encode/decode/forward support and a late-Viewer policy before inclusion; - selecting a codec cannot transcode an existing publication. Confirm live - changes, relay/SFU paths, older 2.x capability subsets and explicit 1.x - mismatch before accepting the replacement contract. The published 1.x enums - remain that major's baseline; the PR is not a compatible enum-only extension + Additional codecs are outside this candidate. Later inclusion needs complete + encode/decode/forward support and a late-Viewer policy; + selecting a codec cannot transcode an existing publication. Final package + acceptance must retain the verified live-change, relay/SFU and limited-receiver + behavior, and check explicit 1.x mismatch with packaged builds. The published + 1.x enums remain that major's baseline; the PR is not a compatible enum-only extension of v1.9.2. - [ ] **2.0 candidate acceptance.** Finish the combined numeric-media UI, App/room lifecycle, ordinary Browser/Native routes and regression checks before From 546e72c0dff2911b4fe7ae08f416bbbe39abaa63 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:52:01 +0800 Subject: [PATCH 151/262] fix(media): separate wire values from runtime admission Preserve valid quality observations without letting rejected telemetry retire media control. Keep numeric requests and capability descriptions independent of local allocation limits, enforce admission before applying capture changes, and remove count-specific retry-progress negotiation. --- docs/research/native-client-media.md | 26 ++++- docs/standards/engineering.md | 31 ++++-- docs/standards/media-quality.md | 15 ++- docs/standards/versioning.md | 6 ++ docs/todo.md | 22 ++++- internal/app/app.go | 2 + .../app/config/quality_preference_test.go | 2 +- internal/app/loopback/server.go | 2 + internal/app/nativecapture/protocol.go | 3 +- internal/app/nativecapture/stream.go | 6 +- internal/app/nativecontrol/session.go | 6 ++ internal/app/nativecontrol/session_test.go | 14 +++ internal/server/app/app_test.go | 8 +- internal/server/app/routes.go | 11 +-- internal/server/protocol/client.go | 58 +++++------ internal/server/protocol/contract_test.go | 23 +---- internal/server/protocol/sfu.go | 4 +- internal/server/protocol/types.go | 63 ++++++------ internal/server/sfu/media.go | 4 +- internal/server/sfu/media_test.go | 19 ++++ internal/server/signal/server.go | 84 ++++++++-------- internal/server/signal/server_test.go | 90 +++++++---------- internal/server/signal/session.go | 22 ++--- src/client/App.tsx | 17 +--- .../components/living/QualityPresets.tsx | 9 +- src/client/lib/api.ts | 14 --- src/client/lib/quality-preference.ts | 3 +- src/client/media/quality.ts | 26 ++++- src/client/media/sender-quality-evidence.ts | 58 +++++------ src/client/media/viewer-quality-evidence.ts | 99 +++++-------------- src/client/native/client.ts | 7 ++ src/client/native/wire.ts | 40 ++++---- src/client/pages/HostPage.tsx | 28 +++--- src/client/pages/ViewerPage.tsx | 5 +- src/shared/protocol.ts | 92 ++++++++--------- src/shared/quality-evidence.ts | 13 +++ tests/client-state.test.ts | 29 ------ tests/fixtures/wire-samples.json | 88 ++++++++++++++++- tests/host-quality-ownership.test.ts | 25 ++++- tests/native-client.test.ts | 47 +++++++++ tests/protocol.test.ts | 54 ++++++---- tests/quality-preference.test.ts | 8 ++ tests/quality.test.ts | 22 ++++- tests/sender-quality-evidence.test.ts | 29 ++++++ tests/viewer-quality-evidence.test.ts | 17 +++- 45 files changed, 730 insertions(+), 521 deletions(-) create mode 100644 src/shared/quality-evidence.ts diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 469907aab..04152e8d2 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -588,7 +588,7 @@ presets. A source ceiling must also yield a valid half-size output. Bitrate and output-count bounds remain unchanged. Candidate signaling and Native control now carry numeric width/height ceilings, with custom input beside the existing UI shortcuts. Descriptive wire bounds are independent of the current implementation maximum; -App discovery advertises axis, pixel-count and macroblock-rate limits, and Native rejects unsupported +App discovery advertises size, pixel-count, macroblock-rate, frame-rate and bitrate limits, and Native rejects unsupported size/rate combinations as operation failures without retiring the control session. Removing an enum is not permission to allocate arbitrary media resources. @@ -706,7 +706,7 @@ ceilings without a fixed aspect ratio or rotation listener, following the Media Capture distinction below. Synthetic devices do not establish physical phone orientation behavior. -The proposed boundary follows the +The boundary follows the [Media Capture specification](https://www.w3.org/TR/mediacapture-streams/#constrainable-interface): requested constraints, supported capabilities and actual settings have different owners. Capability ranges cannot describe every combination of resolution and @@ -714,7 +714,27 @@ frame rate; [WebRTC codec discovery](https://www.w3.org/TR/webrtc/#rtcrtpsender- is optimistic and reserves no hardware resources. An advertised capability is therefore not proof that a particular encoder configuration can start. -| Meaning | Proposed Piik owner and representation | +Wire representation follows the platform types for dimensions/bitrates and a +positive finite frame rate; current capture admission remains narrower. This +allows a reader to reject an unsupported operation without rejecting its room +or control session. Fractional rates must not be silently truncated by Native's +integer capture API. SFU metadata passes the same representation boundary and +then its existing bounded admission before allocation. Capability descriptions +may exceed local support, but cannot authorize larger local allocations. +This separation also appears in +[mediasoup RTP parameters and capabilities](https://mediasoup.org/documentation/v3/mediasoup/rtp-parameters-and-capabilities/). + +[WebRTC statistics](https://www.w3.org/TR/webrtc-stats/#dom-rtcicecandidatepairstats-availableoutgoingbitrate) +describe estimated available capacity, not a configured send-rate ceiling; +[Pion exposes it as a floating-point observation](https://github.com/pion/ice/blob/main/stats.go). +Local in-memory Native control checks reproduced a 100 Mbps observation ceiling +closing the entire connection. The corrected readers preserve high valid +estimates, normalize invalid optional measurements and discard invalid quality +samples without granting them control authority. Shared Go/TypeScript fixtures +also cover sender, publication and receiver evidence. This establishes boundary +behavior, not the cause of an unmatched field report or an improvement to ICE. + +| Meaning | Piik owner and representation | | --- | --- | | Quick presets | UI-local labels mapping to numeric picture/audio requests; adding a label does not extend the wire | | Host intent | One validated numeric size, frame-rate and bitrate request, distinct from measured output | diff --git a/docs/standards/engineering.md b/docs/standards/engineering.md index f59b0501f..e1152c193 100644 --- a/docs/standards/engineering.md +++ b/docs/standards/engineering.md @@ -79,14 +79,29 @@ the relevant existing fixture/check together. Do not create another handwritten field list in documentation, parallel DTO hierarchy or schema generator without a demonstrated reduction in ownership or drift. -Keep UI presets, requested values, runtime capabilities and hard resource limits -distinct. A current menu is not sufficient reason to freeze its values into every -wire reader or platform adapter. Preserve meaningful authorization and resource -bounds; prove that a producer's valid aggregate output fits its consumer, including -serialization overhead. A per-item limit alone does not establish that contract. - -When extending a boundary, exercise a representative new value and an unchanged -consumer before calling the design extensible. For media, include another source +Keep UI presets, requested values, runtime capabilities, observations and hard +resource limits distinct. Every bound needs an owner and a reason: wire/codec +representation, accepted product policy or resource protection. A current menu +or retry budget is not sufficient reason to freeze its values into a wire reader. +Validate representation at the interface, then supported operations at their +owner before allocating or changing live state. An unsupported setting should +use the existing operation rejection, without retiring an otherwise valid session. + +A capability description may exceed the reader's implementation. Intersect it +with local support and budgets; do not treat a stronger peer as absent or grant +resources from its declaration alone. Measurements are observations, not resource +requests: their range need not match a setting or estimated network budget. +An unusable optional metric should become unavailable without discarding valid +evidence or closing media control. Identity, generation, authority and required +state consistency remain strict; never clamp a measurement into an invented fact. + +Preserve meaningful authorization and resource bounds; prove that a producer's +valid aggregate output fits its consumer, including serialization overhead. +A per-item limit alone does not establish that contract. + +When extending a boundary, exercise a representative new value, a larger peer +capability and an unchanged consumer before calling the design extensible. +Check both acceptance and the effect of rejection. For media, include another source orientation and distinguish encode, decode and forwarding support. Optional presentation additions need a bounded unsupported-value policy separate from authority violations, following [versioning](./versioning.md#extending-interfaces). diff --git a/docs/standards/media-quality.md b/docs/standards/media-quality.md index 8a2eba030..75ddae3c6 100644 --- a/docs/standards/media-quality.md +++ b/docs/standards/media-quality.md @@ -137,6 +137,12 @@ Width and height form one applied edit. A size change caps the requested frame rate in the same quality operation when the larger size requires a lower limit. Larger wire-valid requests do not authorize an adapter to exceed its local capture, codec or allocation limits. +Numeric requests use representational bounds rather than the current control +ranges. Current capture still admits integer 15..60 fps and 2..12 Mbps, further +limited by picture size. App descriptions may advertise stronger capabilities; +controls use their intersection with local support. An unsupported request is +an operation failure, not malformed signaling. An unsupported saved preference +does not seed capture or overwrite its stored value with a fallback. Frame rate, bitrate, and `maintain-resolution | balanced | maintain-framerate` retain their controls. Display video uses the standard `contentHint = "motion"` for game motion. @@ -157,11 +163,18 @@ Level 5.1: 983,040 16x16 macroblocks per second, with at most 60 fps. This admit landscape/portrait 4K30 and existing 1440p60, not 4K60. Decoder alignment has a separate coded-storage bound; reference-picture storage is bounded before Native derivation. These are admission limits, not a device performance promise. -App discovery reports axis, pixel-count and output-rate limits through the +App discovery reports size, pixel-count, frame-rate and bitrate limits through the existing native health description. Controls use the intersection with local implementation limits; a remote claim cannot raise those limits. Whole-route and physical-device acceptance remain separate from available controls. +Quality observations have independent numeric bounds. A valid capacity estimate +above the requested send rate remains evidence; invalid optional measurements +become unavailable. Invalid Native quality samples are discarded without closing +media control. Required identity, generation and state evidence must still pass +before a sample can influence route decisions. SFU encoding descriptions likewise +do not grant resources: the SFU admits supported formats before allocating them. + Framework-driven downscaling under bandwidth or device pressure is valid within the selected degradation preference. A low decoded resolution alone is not a defect and does not justify a new quality floor. Review avoidable disruption, diff --git a/docs/standards/versioning.md b/docs/standards/versioning.md index 5b86801b5..34650a258 100644 --- a/docs/standards/versioning.md +++ b/docs/standards/versioning.md @@ -81,6 +81,12 @@ execute unknown commands. A more tolerant reader does not update already-release readers: do not emit a new reaction to legacy pages merely because current pages can skip it. Verify both readers before calling a change additive. +In the unpublished 2.0 contract, numeric media descriptions and retry progress +are independent of current implementation limits and retry policy. Extending +those policies does not require another count-specific capability flag. +Representation acceptance still requires local operation/resource admission; +it is not a promise that every same-major peer can produce a new media format. + Native discovery distinguishes an incompatible Piik App from an absent App and reports protocol numbers to local diagnostics; the source picker offers update/ refresh guidance and Browser operation. Do not reload, reinstall or interrupt diff --git a/docs/todo.md b/docs/todo.md index 601f8a052..bd4c8efa3 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -16,6 +16,21 @@ Git/PRs own completed history. Prepare one coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. +- [ ] **Linux Portal target identity (#470).** Review of the + [contributor fix](https://github.com/TNTcraftHIM/Piik/pull/470) and upstream + PipeWire confirms that the current fallback passes a node ID as an object + serial. Integrate the fix with attribution, retain serial targeting when + available, and use the node-ID selector otherwise. Run the Linux native + checks and verify a Portal that omits `pipewire-serial`; this fix is not yet + in the candidate. +- [ ] **Reconcile open PRs with the final candidate.** + [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s 256/320 kbps request and + [#469](https://github.com/TNTcraftHIM/Piik/pull/469)'s 4K request overlap the + numeric-media candidate; neither PR has been merged. The latter's 4K60 and + 40 Mbps expansion needs separate evidence and is not accepted scope. Preserve + contributor credit and explain the disposition at integration. Rewrite + [#459](https://github.com/TNTcraftHIM/Piik/pull/459)'s earlier maintenance scope + and compatibility claims around the final 2.0 result before integration. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance are complete. Integrate through the next owner-approved publishing merge and verify the separate workflow with its original CI artifacts. Mirror failure @@ -56,7 +71,12 @@ Implementation needs a concrete useful result within the product model. - [ ] **Media parameters and capabilities for 2.0.** Follow the [media assessment](./research/native-client-media.md#media-capability-extension-assessment): separate UI presets, numeric requests, bounded implementation capabilities and - actual output. Numeric audio and video controls are implemented; the assessment + actual output. Numeric request, capability and observation boundaries are now + separate from implementation admission. Keep receiver, codec/alignment and + resource budgets intact; opening an output range still needs end-to-end evidence. + Assess higher audio output against packet-duration/size and receiver support; + removing the current 320 kbps sender guard alone is not sufficient. Numeric + audio and video controls are implemented; the assessment owns local capture, codec, derived-output, SFU and bounded mixed-route evidence. Remaining receiver acceptance needs physical limited decoders and camera orientation behavior, beyond the bounded SDP-limit and mixed-route controls. diff --git a/internal/app/app.go b/internal/app/app.go index 6966ecca4..e112ad100 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -633,6 +633,8 @@ func discoverNativeMedia(ctx context.Context, configuredPath string) nativeRunti }, MaxVideoPixels: nativecapture.MaxVideoPixels, MaxVideoMacroblocksPerSecond: nativecapture.MaxVideoMacroblocksPerSecond, + MaxVideoFramerate: nativecapture.MaxVideoFramerate, + MaxVideoBitrate: nativecapture.MaxVideoBitrate, MaxAudioBitrate: nativeaudio.MaxSendBitrate, ReceiverReuse: true, Video: summary.Video, diff --git a/internal/app/config/quality_preference_test.go b/internal/app/config/quality_preference_test.go index 71b00c366..bab757deb 100644 --- a/internal/app/config/quality_preference_test.go +++ b/internal/app/config/quality_preference_test.go @@ -38,7 +38,7 @@ func TestQualityPreferencePersistsWithoutChangingAppConfiguration(t *testing.T) t.Fatal("preference changed strict App configuration") } invalid := quality - invalid.Resolution.Width = 8194 + invalid.Resolution.Width = 0x1_0000_0000 if p.Save(invalid) == nil || *p.Read() != quality { t.Fatal("invalid choice changed preference") } diff --git a/internal/app/loopback/server.go b/internal/app/loopback/server.go index 7fa232116..0e56477c1 100644 --- a/internal/app/loopback/server.go +++ b/internal/app/loopback/server.go @@ -76,6 +76,8 @@ type NativeMediaCapabilities struct { MaxVideoResolution protocol.QualityResolution `json:"maxVideoResolution,omitzero"` MaxVideoPixels int `json:"maxVideoPixels,omitempty"` MaxVideoMacroblocksPerSecond int `json:"maxVideoMacroblocksPerSecond,omitempty"` + MaxVideoFramerate int `json:"maxVideoFramerate,omitempty"` + MaxVideoBitrate int `json:"maxVideoBitrate,omitempty"` MaxAudioBitrate int `json:"maxAudioBitrate,omitempty"` ReceiverReuse bool `json:"receiverReuse,omitempty"` Video bool `json:"video"` diff --git a/internal/app/nativecapture/protocol.go b/internal/app/nativecapture/protocol.go index e38dc0c53..b3cd65741 100644 --- a/internal/app/nativecapture/protocol.go +++ b/internal/app/nativecapture/protocol.go @@ -18,6 +18,7 @@ const ( MaxVideoDimension = 3840 MaxVideoPixels = 3840 * 2160 MaxVideoMacroblocksPerSecond = 983_040 + MaxVideoFramerate = 60 // Coded pictures may include alignment outside the visible crop. Keep this // separate from visible dimensions and within H.264 Level 5.1 storage. MaxCodedVideoDimension = 4096 @@ -34,7 +35,7 @@ func ValidVideoSize(width, height uint32) bool { // ValidVideoFormat keeps both current encoders within the shared receive // ceiling (H.264 Level 5.1), including non-preset shapes and frame rates. func ValidVideoFormat(width, height, framerate uint32) bool { - return ValidVideoSize(width, height) && framerate >= 1 && framerate <= 60 && + return ValidVideoSize(width, height) && framerate >= 1 && framerate <= MaxVideoFramerate && uint64((width+15)/16)*uint64((height+15)/16)*uint64(framerate) <= MaxVideoMacroblocksPerSecond } diff --git a/internal/app/nativecapture/stream.go b/internal/app/nativecapture/stream.go index f8626c602..5490bccb6 100644 --- a/internal/app/nativecapture/stream.go +++ b/internal/app/nativecapture/stream.go @@ -24,7 +24,7 @@ const ( maxPreviewBytes = 192 * 1024 maxOutputs = 6 minOutputBitrate = 1_000 // Codec rate APIs use whole kbps. - maxOutputBitrate = 12_000_000 + MaxVideoBitrate = 12_000_000 maxSources = 1024 maxSourceTitleBytes = 4096 // Leave room for the control envelope within loopback's 256 KiB message. @@ -89,7 +89,7 @@ type OutputProfile struct { func (profile OutputProfile) Valid() bool { return ValidVideoFormat(profile.Width, profile.Height, profile.Framerate) && - profile.Bitrate >= minOutputBitrate && profile.Bitrate <= maxOutputBitrate + profile.Bitrate >= minOutputBitrate && profile.Bitrate <= MaxVideoBitrate } // ScreenShareOutputs follows LiveKit 2.22.1's screen-share simulcast construction, @@ -112,7 +112,7 @@ func (profile VideoProfile) Valid() bool { // belong to the UI, not the capture process's numeric resource boundary. return ValidVideoFormat(profile.Width, profile.Height, profile.Framerate) && profile.Width >= 4 && profile.Height >= 4 && profile.Framerate >= 15 && - profile.Bitrate >= 2_000_000 && profile.Bitrate <= 12_000_000 && + profile.Bitrate >= 2_000_000 && profile.Bitrate <= MaxVideoBitrate && validVideoPreference(profile.Preference) } diff --git a/internal/app/nativecontrol/session.go b/internal/app/nativecontrol/session.go index ca1af8cf7..85d43bd0c 100644 --- a/internal/app/nativecontrol/session.go +++ b/internal/app/nativecontrol/session.go @@ -643,6 +643,12 @@ func (session *Session) startShare( } func nativeQualityProfile(settings qualitySettings) nativehost.QualityProfile { + // The wire can express more than this integer capture API. Never truncate a + // fractional rate or wrap a wider value into an apparently supported profile. + if settings.MaxFramerate < 1 || settings.MaxFramerate > 0xffff_ffff || + settings.MaxFramerate != protocol.Num(uint32(settings.MaxFramerate)) { + return nativehost.QualityProfile{} + } audioBitrate := int(settings.MaxAudioBitrate) if audioBitrate == 0 { audioBitrate = int(protocol.DefaultQualitySettings.MaxAudioBitrate) diff --git a/internal/app/nativecontrol/session_test.go b/internal/app/nativecontrol/session_test.go index 4e3f633a7..51868f6db 100644 --- a/internal/app/nativecontrol/session_test.go +++ b/internal/app/nativecontrol/session_test.go @@ -677,6 +677,12 @@ func TestQualitySettingsMapOnceIntoNativeMedia(t *testing.T) { if !nativeQualityProfile(settings).Valid() { t.Fatal("4K30 was rejected") } + for _, fps := range []protocol.Num{30.5, 120, 0x1_0000_0000 + 30} { + settings.MaxFramerate = fps + if nativeQualityProfile(settings).Valid() { + t.Fatalf("unsupported fps %v was narrowed into a valid profile", fps) + } + } } func TestUpdateShareRequiresTheCurrentStrictProfile(t *testing.T) { @@ -690,6 +696,14 @@ func TestUpdateShareRequiresTheCurrentStrictProfile(t *testing.T) { assertOperationFailure(t, session, strings.Replace(valid, `"resolution":{"width":1920,"height":1080}`, `"resolution":{"width":7680,"height":4320}`, 1)) + for _, replace := range [][2]string{ + {`"width":1920`, `"width":32768`}, {`"width":1920`, `"width":1921`}, + {`"maxFramerate":30`, `"maxFramerate":120`}, {`"maxFramerate":30`, `"maxFramerate":30.5`}, + {`"maxFramerate":30`, `"maxFramerate":4294967326`}, {`"maxFramerate":30`, `"maxFramerate":1`}, + {`"maxBitrate":5000000`, `"maxBitrate":40000000`}, {`"maxBitrate":5000000`, `"maxBitrate":1000`}, + } { + assertOperationFailure(t, session, strings.Replace(valid, replace[0], replace[1], 1)) + } if _, err := session.Handle(t.Context(), []byte(`{"version":10,"id":"request_options","type":"capture-options"}`)); err != nil { t.Fatalf("unsupported media setting retired control: %v", err) } diff --git a/internal/server/app/app_test.go b/internal/server/app/app_test.go index e0b9347d3..8533421b0 100644 --- a/internal/server/app/app_test.go +++ b/internal/server/app/app_test.go @@ -366,7 +366,7 @@ func TestCapabilitiesReportServicesDisabledByDefault(t *testing.T) { server := start(t, Options{Config: testConfig(t)}) server.do(http.MethodGet, "/api/capabilities"). - expect(http.StatusOK, `{"connectionAttemptProgress4":true,"sfu":false,"natPrediction":false,"roomInteractions":true,"hostRoomSession":true}`) + expect(http.StatusOK, `{"sfu":false,"natPrediction":false,"roomInteractions":true,"hostRoomSession":true}`) } @@ -376,7 +376,7 @@ func TestCapabilitiesReportSFUOnlyPolicy(t *testing.T) { configuration.NATPredictionEnabled = true server := start(t, Options{Config: configuration}) server.do(http.MethodGet, "/api/capabilities"). - expect(http.StatusOK, `{"connectionAttemptProgress4":true,"sfu":true,"sfuOnly":true,"natPrediction":false,"roomInteractions":true,"hostRoomSession":true}`) + expect(http.StatusOK, `{"sfu":true,"sfuOnly":true,"natPrediction":false,"roomInteractions":true,"hostRoomSession":true}`) } func TestCapabilitiesReportOptionalNATPredictionWithoutExposingConfiguration(t *testing.T) { @@ -387,7 +387,7 @@ func TestCapabilitiesReportOptionalNATPredictionWithoutExposingConfiguration(t * server := start(t, Options{Config: configuration}) server.do(http.MethodGet, "/api/capabilities"). - expect(http.StatusOK, `{"connectionAttemptProgress4":true,"sfu":false,"natPrediction":true,"roomInteractions":true,"hostRoomSession":true}`). + expect(http.StatusOK, `{"sfu":false,"natPrediction":true,"roomInteractions":true,"hostRoomSession":true}`). expectHeader("Cache-Control", "no-store"). expectHeader("X-Content-Type-Options", "nosniff") @@ -430,7 +430,7 @@ func TestSFUListenerFollowsApplicationCloseAndEnd(t *testing.T) { t.Fatal("missing runtime owner") } server.do(http.MethodGet, "/api/capabilities"). - expect(http.StatusOK, `{"connectionAttemptProgress4":true,"sfu":true,"natPrediction":false,"roomInteractions":true,"hostRoomSession":true}`) + expect(http.StatusOK, `{"sfu":true,"natPrediction":false,"roomInteractions":true,"hostRoomSession":true}`) var err error if ending { diff --git a/internal/server/app/routes.go b/internal/server/app/routes.go index 7a69e23fb..6d1655781 100644 --- a/internal/server/app/routes.go +++ b/internal/server/app/routes.go @@ -77,12 +77,11 @@ func (s *Server) route( return } sendJSON(writer, http.StatusOK, protocol.RuntimeCapabilities{ - ConnectionAttemptProgress4: true, - Sfu: s.config.SFU != nil, - SfuOnly: s.config.SFU != nil && s.config.SFU.Only, - NatPrediction: s.config.NATPredictionEnabled && !(s.config.SFU != nil && s.config.SFU.Only), - RoomInteractions: true, - HostRoomSession: true, + Sfu: s.config.SFU != nil, + SfuOnly: s.config.SFU != nil && s.config.SFU.Only, + NatPrediction: s.config.NATPredictionEnabled && !(s.config.SFU != nil && s.config.SFU.Only), + RoomInteractions: true, + HostRoomSession: true, }) return } diff --git a/internal/server/protocol/client.go b/internal/server/protocol/client.go index ae318ab96..ba004829c 100644 --- a/internal/server/protocol/client.go +++ b/internal/server/protocol/client.go @@ -13,35 +13,33 @@ type ClientMessage interface { // AuthenticateHostMessage is the host member of authenticateMessageSchema. type AuthenticateHostMessage struct { - Type string `json:"type"` - Protocol string `json:"protocol"` - RoomID string `json:"roomId"` - Role Role `json:"role"` - Token string `json:"token"` - ClientID string `json:"clientId"` - ShareGeneration string `json:"shareGeneration,omitempty"` - SharingPaused *bool `json:"sharingPaused,omitempty"` - QualitySettings *QualitySettings `json:"qualitySettings,omitempty"` - RoutePolicy RoutePolicy `json:"routePolicy"` - ViewerPresence bool `json:"viewerPresence,omitempty"` - ConnectionAttemptProgress4 bool `json:"connectionAttemptProgress4,omitempty"` - DisplayName *DisplayName `json:"displayName,omitempty"` - RoomOnly bool `json:"roomOnly,omitempty"` - RoomSession bool `json:"roomSession,omitempty"` + Type string `json:"type"` + Protocol string `json:"protocol"` + RoomID string `json:"roomId"` + Role Role `json:"role"` + Token string `json:"token"` + ClientID string `json:"clientId"` + ShareGeneration string `json:"shareGeneration,omitempty"` + SharingPaused *bool `json:"sharingPaused,omitempty"` + QualitySettings *QualitySettings `json:"qualitySettings,omitempty"` + RoutePolicy RoutePolicy `json:"routePolicy"` + ViewerPresence bool `json:"viewerPresence,omitempty"` + DisplayName *DisplayName `json:"displayName,omitempty"` + RoomOnly bool `json:"roomOnly,omitempty"` + RoomSession bool `json:"roomSession,omitempty"` } // AuthenticateViewerMessage is the viewer member of authenticateMessageSchema. type AuthenticateViewerMessage struct { - Type string `json:"type"` - Protocol string `json:"protocol"` - RoomID string `json:"roomId"` - Role Role `json:"role"` - ClientID string `json:"clientId"` - ViewerGrant string `json:"viewerGrant,omitempty"` - ViewerPassword string `json:"viewerPassword,omitempty"` - DisplayName *DisplayName `json:"displayName,omitempty"` - ViewerPresence bool `json:"viewerPresence,omitempty"` - ConnectionAttemptProgress4 bool `json:"connectionAttemptProgress4,omitempty"` + Type string `json:"type"` + Protocol string `json:"protocol"` + RoomID string `json:"roomId"` + Role Role `json:"role"` + ClientID string `json:"clientId"` + ViewerGrant string `json:"viewerGrant,omitempty"` + ViewerPassword string `json:"viewerPassword,omitempty"` + DisplayName *DisplayName `json:"displayName,omitempty"` + ViewerPresence bool `json:"viewerPresence,omitempty"` } // SignalingChallengeMessage is { type: "signaling-challenge", sequence }. @@ -301,7 +299,7 @@ func decodeAuthenticate(data []byte) (ClientMessage, error) { return nil, err } if err := present.optional("shareGeneration", "roomOnly", "roomSession", "sharingPaused", "qualitySettings", - "routePolicy", "viewerPresence", "displayName", "connectionAttemptProgress4"); err != nil { + "routePolicy", "viewerPresence", "displayName"); err != nil { return nil, err } @@ -332,9 +330,6 @@ func decodeAuthenticate(data []byte) (ClientMessage, error) { if present.has("viewerPresence") && !message.ViewerPresence { return nil, errors.New("viewerPresence must be true when present") } - if present.has("connectionAttemptProgress4") && !message.ConnectionAttemptProgress4 { - return nil, errors.New("connectionAttemptProgress4 must be true when present") - } if !present.has("routePolicy") { message.RoutePolicy = DefaultRoutePolicy } @@ -349,7 +344,7 @@ func decodeAuthenticate(data []byte) (ClientMessage, error) { return nil, err } if err := present.optional( - "viewerGrant", "viewerPassword", "displayName", "viewerPresence", "connectionAttemptProgress4"); err != nil { + "viewerGrant", "viewerPassword", "displayName", "viewerPresence"); err != nil { return nil, err } if message.Protocol != SignalingProtocol { @@ -370,9 +365,6 @@ func decodeAuthenticate(data []byte) (ClientMessage, error) { if present.has("viewerPresence") && !message.ViewerPresence { return nil, errors.New("viewerPresence must be true when present") } - if present.has("connectionAttemptProgress4") && !message.ConnectionAttemptProgress4 { - return nil, errors.New("connectionAttemptProgress4 must be true when present") - } return message, nil } return nil, fmt.Errorf("unknown authenticate role %q", role) diff --git a/internal/server/protocol/contract_test.go b/internal/server/protocol/contract_test.go index 3ebd15393..9f37b1be7 100644 --- a/internal/server/protocol/contract_test.go +++ b/internal/server/protocol/contract_test.go @@ -14,7 +14,6 @@ func TestRuntimeCapabilities(t *testing.T) { want RuntimeCapabilities }{ {`{}`, RuntimeCapabilities{}}, - {`{"connectionAttemptProgress4":true}`, RuntimeCapabilities{ConnectionAttemptProgress4: true}}, {`{"sfu":true,"extra":{"enabled":true}}`, RuntimeCapabilities{Sfu: true}}, {`{"sfu":true,"sfuOnly":true}`, RuntimeCapabilities{Sfu: true, SfuOnly: true}}, {`{"natPrediction":true,"SFU":true}`, RuntimeCapabilities{NatPrediction: true}}, @@ -29,7 +28,6 @@ func TestRuntimeCapabilities(t *testing.T) { for _, data := range []string{ `null`, `[]`, `true`, `{"sfu":null}`, `{"sfu":"true"}`, `{"sfuOnly":null}`, `{"sfuOnly":"true"}`, `{"natPrediction":null}`, `{"natPrediction":1}`, - `{"connectionAttemptProgress4":null}`, `{"connectionAttemptProgress4":4}`, } { var got RuntimeCapabilities if err := json.Unmarshal([]byte(data), &got); err == nil { @@ -243,9 +241,9 @@ func TestAuthenticatedRoomRevision(t *testing.T) { object(drop(authenticatedHost(), "routeAssignment")), false) assertServerMessage(t, "absent media mode", object(drop(authenticatedHost(), "mediaMode")), false) - assertServerMessage(t, "quality settings past the bitrate ceiling", + assertServerMessage(t, "quality settings past the numeric representation", object(with(authenticatedHost(), member{"qualitySettings", - `{"resolution":{"width":1920,"height":1080},"maxFramerate":60,"maxBitrate":20000000,` + + `{"resolution":{"width":1920,"height":1080},"maxFramerate":60,"maxBitrate":4294967296,` + `"degradationPreference":"maintain-resolution"}`})), false) } @@ -499,7 +497,9 @@ func TestPreparedRouteCandidate(t *testing.T) { {`{"current":1,"total":4}`, true}, {`{"current":4,"total":4}`, true}, {`{"current":5,"total":4}`, false}, - {`{"current":1,"total":5}`, false}, + {`{"current":1,"total":5}`, true}, + {`{"current":1,"total":0}`, false}, + {`{"current":1,"total":9007199254740992}`, false}, {`{"current":1.5,"total":3}`, false}, } { assertServerMessage(t, "connectionAttempt "+sample.attempt, @@ -690,19 +690,6 @@ func TestOptionalAndNullableAreExact(t *testing.T) { } // A host authenticate without routePolicy takes DEFAULT_ROUTE_POLICY. -func TestAuthenticateConnectionProgressOptIn(t *testing.T) { - for _, role := range []string{"host", "viewer"} { - base := `"type":"authenticate","protocol":"` + SignalingProtocol + `","roomId":"1234","clientId":"client_12345678","role":"` + role + `"` - if role == "host" { - base += `,"token":"` + repeat("a", 43) + `"` - } - for _, value := range []string{"true", "false", "null", "4", `"true"`} { - assertClientMessage(t, role+value, []byte(`{`+base+`,"connectionAttemptProgress4":`+value+`}`), value == "true") - } - assertClientMessage(t, role+" baseline", []byte(`{`+base+`}`), true) - } -} - func TestAuthenticateAppliesTheRoutePolicyDefault(t *testing.T) { message, err := DecodeClientMessage([]byte( `{"type":"authenticate","protocol":"piik-v24","roomId":"1234","role":"host",` + diff --git a/internal/server/protocol/sfu.go b/internal/server/protocol/sfu.go index 5e4000474..f7691a11e 100644 --- a/internal/server/protocol/sfu.go +++ b/internal/server/protocol/sfu.go @@ -27,12 +27,12 @@ func (value *SfuMedia) UnmarshalJSON(data []byte) error { return err } if !enumOf(value.Codec, "h264", "vp8") || len(value.Layers) < 1 || len(value.Layers) > 3 || - !inRangeInt(value.AudioBitrate, 0, 510_000) || value.Audio && value.AudioBitrate == 0 { + !inRangeInt(value.AudioBitrate, 0, MaxAudioBitrate) || value.Audio && value.AudioBitrate == 0 { return errors.New("invalid SFU media") } for index, layer := range value.Layers { if len(layer.RID) > 16 || layer.RID == "" && len(value.Layers) != 1 || - !inRangeInt(layer.Width, 1, 8192) || !inRangeInt(layer.Height, 1, 8192) || !inRangeInt(layer.Bitrate, 1, 100_000_000) { + !inRangeInt(layer.Width, 1, 0xffff_ffff) || !inRangeInt(layer.Height, 1, 0xffff_ffff) || !inRangeInt(layer.Bitrate, 1, 0xffff_ffff) { return errors.New("invalid SFU layer") } for _, previous := range value.Layers[:index] { diff --git a/internal/server/protocol/types.go b/internal/server/protocol/types.go index f201cc158..88542246a 100644 --- a/internal/server/protocol/types.go +++ b/internal/server/protocol/types.go @@ -120,8 +120,7 @@ func (v *QualityResolution) UnmarshalJSON(data []byte) error { if err := present.require("width", "height"); err != nil { return err } - if !inRangeInt(v.Width, 4, 8192) || v.Width%2 != 0 || - !inRangeInt(v.Height, 4, 8192) || v.Height%2 != 0 { + if !inRangeInt(v.Width, 1, 0xffff_ffff) || !inRangeInt(v.Height, 1, 0xffff_ffff) { return errors.New("resolution is out of range") } return nil @@ -130,7 +129,7 @@ func (v *QualityResolution) UnmarshalJSON(data []byte) error { // QualitySettings mirrors qualitySettingsSchema. type QualitySettings struct { Resolution QualityResolution `json:"resolution"` - MaxFramerate Int `json:"maxFramerate"` + MaxFramerate Num `json:"maxFramerate"` MaxBitrate Int `json:"maxBitrate"` DegradationPreference string `json:"degradationPreference"` MaxAudioBitrate Int `json:"maxAudioBitrate,omitempty"` @@ -151,10 +150,10 @@ func (v *QualitySettings) UnmarshalJSON(data []byte) error { if err := present.optional("maxAudioBitrate"); err != nil { return err } - if !inRangeInt(v.MaxFramerate, 15, 60) { + if v.MaxFramerate <= 0 || !inRangeNum(v.MaxFramerate, 0, MaxSafeInteger) { return errors.New("maxFramerate is out of range") } - if !inRangeInt(v.MaxBitrate, 2_000_000, 12_000_000) { + if !inRangeInt(v.MaxBitrate, 1, 0xffff_ffff) { return errors.New("maxBitrate is out of range") } if !enumOf(v.DegradationPreference, degradationPreferences...) { @@ -186,12 +185,11 @@ func (v *RoutePolicy) UnmarshalJSON(data []byte) error { // RuntimeCapabilities mirrors runtimeCapabilitiesSchema. type RuntimeCapabilities struct { - ConnectionAttemptProgress4 bool `json:"connectionAttemptProgress4,omitempty"` - Sfu bool `json:"sfu"` - SfuOnly bool `json:"sfuOnly,omitempty"` - NatPrediction bool `json:"natPrediction"` - RoomInteractions bool `json:"roomInteractions,omitempty"` - HostRoomSession bool `json:"hostRoomSession,omitempty"` + Sfu bool `json:"sfu"` + SfuOnly bool `json:"sfuOnly,omitempty"` + NatPrediction bool `json:"natPrediction"` + RoomInteractions bool `json:"roomInteractions,omitempty"` + HostRoomSession bool `json:"hostRoomSession,omitempty"` } // UnmarshalJSON implements json.Unmarshaler. @@ -204,14 +202,13 @@ func (v *RuntimeCapabilities) UnmarshalJSON(data []byte) error { if present == nil { return errors.New("runtime capabilities must be an object") } - if err := present.optional("sfu", "sfuOnly", "roomInteractions", "hostRoomSession", "natPrediction", "connectionAttemptProgress4"); err != nil { + if err := present.optional("sfu", "sfuOnly", "roomInteractions", "hostRoomSession", "natPrediction"); err != nil { return err } for key, target := range map[string]*bool{ "sfu": &v.Sfu, "sfuOnly": &v.SfuOnly, "natPrediction": &v.NatPrediction, - "connectionAttemptProgress4": &v.ConnectionAttemptProgress4, - "roomInteractions": &v.RoomInteractions, "hostRoomSession": &v.HostRoomSession, + "roomInteractions": &v.RoomInteractions, "hostRoomSession": &v.HostRoomSession, } { if value, ok := present[key]; ok { if err := json.Unmarshal(value, target); err != nil { @@ -510,8 +507,8 @@ func (v *ConnectionAttempt) UnmarshalJSON(data []byte) error { if !inRangeInt(v.Current, 1, int64(v.Total)) { return errors.New("connectionAttempt.current is out of range") } - if v.Total != 3 && v.Total != 4 { - return errors.New("connectionAttempt.total must be 3 or 4") + if !inRangeInt(v.Total, 1, MaxSafeInteger) { + return errors.New("connectionAttempt.total is out of range") } return nil } @@ -997,11 +994,11 @@ func (v *ViewerQualityEvidenceMetrics) UnmarshalJSON(data []byte) error { value *Int minimum, maximum int64 }{ - {v.Width, 1, 16_384}, {v.Height, 1, 16_384}, - {v.PacketsReceivedDelta, 0, 1_000_000}, {v.PacketsLostDelta, 0, 1_000_000}, - {v.FramesDecodedDelta, 0, 10_000}, {v.FramesDroppedDelta, 0, 10_000}, + {v.Width, 1, MaxSafeInteger}, {v.Height, 1, MaxSafeInteger}, + {v.PacketsReceivedDelta, 0, MaxSafeInteger}, {v.PacketsLostDelta, 0, MaxSafeInteger}, + {v.FramesDecodedDelta, 0, MaxSafeInteger}, {v.FramesDroppedDelta, 0, MaxSafeInteger}, {v.FreezeCountDelta, 0, MaxSafeInteger}, {v.PauseCountDelta, 0, MaxSafeInteger}, - {v.AudioConcealmentEventsDelta, 0, 10_000}, + {v.AudioConcealmentEventsDelta, 0, MaxSafeInteger}, } { if bound.value != nil && !inRangeInt(*bound.value, bound.minimum, bound.maximum) { return errors.New("viewer quality evidence integer is out of range") @@ -1011,15 +1008,15 @@ func (v *ViewerQualityEvidenceMetrics) UnmarshalJSON(data []byte) error { value *Num minimum, maximum float64 }{ - {v.FramesPerSecond, 0, 240}, {v.BitrateKbps, 0, 100_000}, - {v.RttMs, 0, 60_000}, {v.JitterMs, 0, 60_000}, - {v.DecodeMsPerFrame, 0, 60_000}, + {v.FramesPerSecond, 0, MaxSafeInteger}, {v.BitrateKbps, 0, MaxSafeInteger}, + {v.RttMs, 0, MaxSafeInteger}, {v.JitterMs, 0, MaxSafeInteger}, + {v.DecodeMsPerFrame, 0, MaxSafeInteger}, {v.FreezeDurationMsDelta, 0, MaxSafeInteger}, {v.PauseDurationMsDelta, 0, MaxSafeInteger}, - {v.AudioBitrateKbps, 0, 10_000}, {v.AudioPacketLossPercent, 0, 100}, - {v.AudioJitterMs, 0, 60_000}, - {v.AudioVideoPlayoutDeltaMs, -60_000, 60_000}, - {v.VideoJitterBufferDelayMs, 0, 60_000}, {v.AudioJitterBufferDelayMs, 0, 60_000}, + {v.AudioBitrateKbps, 0, MaxSafeInteger}, {v.AudioPacketLossPercent, 0, 100}, + {v.AudioJitterMs, 0, MaxSafeInteger}, + {v.AudioVideoPlayoutDeltaMs, -MaxSafeInteger, MaxSafeInteger}, + {v.VideoJitterBufferDelayMs, 0, MaxSafeInteger}, {v.AudioJitterBufferDelayMs, 0, MaxSafeInteger}, {v.AudioConcealedSamplesPercent, 0, 100}, } { if bound.value != nil && !inRangeNum(*bound.value, bound.minimum, bound.maximum) { @@ -1103,11 +1100,11 @@ func (v *SenderQualityDiagnostics) UnmarshalJSON(data []byte) error { value *Num minimum, maximum float64 }{ - {v.FramesPerSecond, 0, 240}, {v.BitrateKbps, 0, 100_000}, - {v.CaptureFramesPerSecond.Value, 0, 240}, - {v.MediaSourceFramesPerSecond.Value, 0, 240}, - {v.AvailableOutgoingKbps.Value, 0, 100_000}, - {v.RttMs.Value, 0, 60_000}, {v.PacketLossPercent.Value, 0, 100}, + {v.FramesPerSecond, 0, MaxSafeInteger}, {v.BitrateKbps, 0, MaxSafeInteger}, + {v.CaptureFramesPerSecond.Value, 0, MaxSafeInteger}, + {v.MediaSourceFramesPerSecond.Value, 0, MaxSafeInteger}, + {v.AvailableOutgoingKbps.Value, 0, MaxSafeInteger}, + {v.RttMs.Value, 0, MaxSafeInteger}, {v.PacketLossPercent.Value, 0, 100}, } { if bound.value != nil && !inRangeNum(*bound.value, bound.minimum, bound.maximum) { return errors.New("sender quality diagnostic number is out of range") @@ -1117,7 +1114,7 @@ func (v *SenderQualityDiagnostics) UnmarshalJSON(data []byte) error { value *Int maximum int64 }{ - {v.Width.Value, 16_384}, {v.Height.Value, 16_384}, + {v.Width.Value, MaxSafeInteger}, {v.Height.Value, MaxSafeInteger}, {v.VideoEncodingCount.Value, MaxSafeInteger}, {v.ActiveVideoEncodingCount.Value, MaxSafeInteger}, } { diff --git a/internal/server/sfu/media.go b/internal/server/sfu/media.go index 4c2be04f7..4c9a6155a 100644 --- a/internal/server/sfu/media.go +++ b/internal/server/sfu/media.go @@ -127,7 +127,9 @@ func validPublicationMedia(metadata PublicationMedia) bool { return false } for index, format := range metadata.Formats { - if format.Width == 0 || format.Height == 0 || format.Bitrate == 0 || + // These are this SFU's admission bounds, not the descriptive wire format. + if format.Width == 0 || format.Width > 8192 || format.Height == 0 || format.Height > 8192 || + format.Bitrate == 0 || format.Bitrate > 100_000_000 || len(metadata.RIDs[index]) > 16 || (metadata.RIDs[index] == "" && len(metadata.RIDs) != 1) || slices.Contains(metadata.RIDs[:index], metadata.RIDs[index]) { diff --git a/internal/server/sfu/media_test.go b/internal/server/sfu/media_test.go index 19b78aa78..9fb61e05c 100644 --- a/internal/server/sfu/media_test.go +++ b/internal/server/sfu/media_test.go @@ -30,6 +30,25 @@ func TestMediaOnePublicationTwoSubscribersAndExactRetirement(t *testing.T) { } } +func TestUnsupportedPublicationDescriptionDoesNotAllocate(t *testing.T) { + service := NewMedia(MediaOptions{}) + defer service.Close() + fence := ResourceFence{RoomID: "1234", ShareGeneration: "share_12345678", PublicationGeneration: "publication_12345678"} + for _, format := range []forwarding.LayerFormat{ + {Width: 32768, Height: 32768, Bitrate: 5_000_000}, + {Width: 1920, Height: 1080, Bitrate: 200_000_000}, + {Width: 0xffff_ffff, Height: 0xffff_ffff, Bitrate: 0xffff_ffff}, + } { + metadata := PublicationMedia{Codec: "vp8", RIDs: []string{""}, Formats: []forwarding.LayerFormat{format}} + if err := service.PreparePublication(fence, "publisher_12345678", metadata); err == nil { + t.Fatal("unsupported format was admitted") + } + if service.HasPublication(fence, "publisher_12345678") { + t.Fatal("unsupported format allocated a publication") + } + } +} + func TestMediaDemandBootstrapsWaitingSubscriberAndStopsAfterExactClose(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) defer cancel() diff --git a/internal/server/signal/server.go b/internal/server/signal/server.go index ca0fc8c25..73a8a0403 100644 --- a/internal/server/signal/server.go +++ b/internal/server/signal/server.go @@ -699,51 +699,48 @@ func (s *Server) handleMessage(sess *session, encoded []byte) { // fields authenticate() and authenticationErrorCode() read. Empty strings // and nil pointers are absent keys. type authRequest struct { - role protocol.Role - roomID string - clientID string - token string - viewerGrant string - viewerPassword string - shareGeneration string - sharingPaused bool - qualitySettings *protocol.QualitySettings - routePolicy protocol.RoutePolicy - viewerPresence bool - connectionAttemptProgress4 bool - displayName *string - roomOnly bool - roomSession bool + role protocol.Role + roomID string + clientID string + token string + viewerGrant string + viewerPassword string + shareGeneration string + sharingPaused bool + qualitySettings *protocol.QualitySettings + routePolicy protocol.RoutePolicy + viewerPresence bool + displayName *string + roomOnly bool + roomSession bool } func authRequestOf(message protocol.ClientMessage) (authRequest, bool) { switch m := message.(type) { case protocol.AuthenticateHostMessage: return authRequest{ - role: protocol.RoleHost, - roomID: m.RoomID, - clientID: m.ClientID, - token: m.Token, - shareGeneration: m.ShareGeneration, - sharingPaused: m.SharingPaused != nil && *m.SharingPaused, - qualitySettings: m.QualitySettings, - routePolicy: m.RoutePolicy, - viewerPresence: m.ViewerPresence, - connectionAttemptProgress4: m.ConnectionAttemptProgress4, - displayName: displayNameString(m.DisplayName), - roomOnly: m.RoomOnly, - roomSession: m.RoomSession, + role: protocol.RoleHost, + roomID: m.RoomID, + clientID: m.ClientID, + token: m.Token, + shareGeneration: m.ShareGeneration, + sharingPaused: m.SharingPaused != nil && *m.SharingPaused, + qualitySettings: m.QualitySettings, + routePolicy: m.RoutePolicy, + viewerPresence: m.ViewerPresence, + displayName: displayNameString(m.DisplayName), + roomOnly: m.RoomOnly, + roomSession: m.RoomSession, }, true case protocol.AuthenticateViewerMessage: return authRequest{ - role: protocol.RoleViewer, - roomID: m.RoomID, - clientID: m.ClientID, - viewerGrant: m.ViewerGrant, - viewerPassword: m.ViewerPassword, - viewerPresence: m.ViewerPresence, - connectionAttemptProgress4: m.ConnectionAttemptProgress4, - displayName: displayNameString(m.DisplayName), + role: protocol.RoleViewer, + roomID: m.RoomID, + clientID: m.ClientID, + viewerGrant: m.ViewerGrant, + viewerPassword: m.ViewerPassword, + viewerPresence: m.ViewerPresence, + displayName: displayNameString(m.DisplayName), }, true } return authRequest{}, false @@ -893,14 +890,13 @@ func (s *Server) completeAuthentication(sess *session, request authRequest, part } displayName := authenticatedDisplayName(request) authenticated := &authenticatedSession{ - roomID: roomID, - role: participant.Role, - peerID: participant.PeerID, - shareGeneration: shareGeneration, - displayName: displayName, - viewerPresence: request.viewerPresence, - connectionAttemptProgress4: request.connectionAttemptProgress4, - roomSession: request.roomSession, + roomID: roomID, + role: participant.Role, + peerID: participant.PeerID, + shareGeneration: shareGeneration, + displayName: displayName, + viewerPresence: request.viewerPresence, + roomSession: request.roomSession, } sess.authenticated = authenticated if participant.Role == protocol.RoleViewer { diff --git a/internal/server/signal/server_test.go b/internal/server/signal/server_test.go index 972767b2d..388a8dc1d 100644 --- a/internal/server/signal/server_test.go +++ b/internal/server/signal/server_test.go @@ -1011,16 +1011,15 @@ func isMediaReady(entry presenceEntry) bool { // --------------------------------------------------------------------------- type presenceOptions struct { - connectionAttemptProgress4 bool - displayName string - viewerPresence bool - viewerPassword string - codeOnly bool - sharingPaused *bool - qualitySettings map[string]any - routePolicy *protocol.RoutePolicy - roomSession bool - roomOnly bool + displayName string + viewerPresence bool + viewerPassword string + codeOnly bool + sharingPaused *bool + qualitySettings map[string]any + routePolicy *protocol.RoutePolicy + roomSession bool + roomOnly bool } // authenticate ports the authenticate() helper. relayCapacity -1 is the TS @@ -1043,9 +1042,6 @@ func authenticate( "role": role, "clientId": clientID, } - if presence.connectionAttemptProgress4 { - message["connectionAttemptProgress4"] = true - } if role == protocol.RoleHost { message["token"] = rm.HostToken if shareGeneration != "" { @@ -1376,51 +1372,31 @@ func TestSignalDeliversOnlySelfHostedAuxiliaryNatStunEndpoints(t *testing.T) { // lifecycle and restart // --------------------------------------------------------------------------- -func TestConnectionAttemptProgressIsNegotiatedForEachRecipient(t *testing.T) { - for _, hostSupport := range []bool{false, true} { - for _, viewerSupport := range []bool{false, true} { - t.Run(fmt.Sprintf("host=%t/viewer=%t", hostSupport, viewerSupport), func(t *testing.T) { - h := startHarness(t, harnessOptions{ - natPredictionEnabled: true, - stunURLs: []string{"stun:share.example.test:3478"}, - }) - host, viewer := openClient(t, h), openClient(t, h) - policy := protocol.RoutePolicy{PeerOnly: true, NatPrediction: true} - authenticate(t, host, h.room, protocol.RoleHost, "progress-host", 1, "progress_share_generation_12345678", - presenceOptions{routePolicy: &policy, connectionAttemptProgress4: hostSupport}) - authenticate(t, viewer, h.room, protocol.RoleViewer, "progress-viewer", 0, "", - presenceOptions{connectionAttemptProgress4: viewerSupport}) - for current := 1; current <= 4; current++ { - hostPrepare, viewerPrepare := nextPreparedRoute(t, host), nextPreparedRoute(t, viewer) - for _, recipient := range []struct { - update routeUpdate - support bool - }{ - {hostPrepare, hostSupport}, {viewerPrepare, viewerSupport}, - } { - decoded, err := protocol.DecodeServerMessage(recipient.update.raw) - if err != nil { - t.Fatal(err) - } - progress := decoded.(protocol.RouteUpdatePrepareMessage).Candidate.ConnectionAttempt - if recipient.support { - if progress == nil || *progress != (protocol.ConnectionAttempt{Current: protocol.Int(current), Total: 4}) { - t.Fatalf("new reader progress = %+v", progress) - } - } else if progress != nil { - t.Fatalf("old reader received unsupported progress: %+v", progress) - } - } - if hostPrepare.Candidate.ConnectionID != viewerPrepare.Candidate.ConnectionID { - t.Fatal("reader capability changed the actual candidate") - } - viewer.sendJSON(map[string]any{ - "type": "route-failed", "phase": "prepare", - "revision": viewerPrepare.Revision, "connectionId": viewerPrepare.Candidate.ConnectionID, - }) - } - }) +func TestConnectionAttemptProgressMatchesBothRecipients(t *testing.T) { + h := startHarness(t, harnessOptions{natPredictionEnabled: true, + stunURLs: []string{"stun:share.example.test:3478"}}) + host, viewer := openClient(t, h), openClient(t, h) + policy := protocol.RoutePolicy{PeerOnly: true, NatPrediction: true} + authenticate(t, host, h.room, protocol.RoleHost, "progress-host", 1, "progress_share_generation_12345678", + presenceOptions{routePolicy: &policy}) + authenticate(t, viewer, h.room, protocol.RoleViewer, "progress-viewer", 0, "", presenceOptions{}) + for current := 1; current <= 4; current++ { + hostPrepare, viewerPrepare := nextPreparedRoute(t, host), nextPreparedRoute(t, viewer) + for _, update := range []routeUpdate{hostPrepare, viewerPrepare} { + decoded, err := protocol.DecodeServerMessage(update.raw) + if err != nil { + t.Fatal(err) + } + progress := decoded.(protocol.RouteUpdatePrepareMessage).Candidate.ConnectionAttempt + if progress == nil || *progress != (protocol.ConnectionAttempt{Current: protocol.Int(current), Total: 4}) { + t.Fatalf("progress = %+v", progress) + } + } + if hostPrepare.Candidate.ConnectionID != viewerPrepare.Candidate.ConnectionID { + t.Fatal("recipients disagree on the candidate") } + viewer.sendJSON(map[string]any{"type": "route-failed", "phase": "prepare", + "revision": viewerPrepare.Revision, "connectionId": viewerPrepare.Candidate.ConnectionID}) } } diff --git a/internal/server/signal/session.go b/internal/server/signal/session.go index ca4585ec9..1d9f2f011 100644 --- a/internal/server/signal/session.go +++ b/internal/server/signal/session.go @@ -16,14 +16,13 @@ import ( // authentication replaced between dispatch and use. Empty shareGeneration // means no share; nil displayName means the participant has no display name. type authenticatedSession struct { - roomID string - role protocol.Role - peerID string - shareGeneration string - displayName *string - viewerPresence bool - connectionAttemptProgress4 bool - roomSession bool + roomID string + role protocol.Role + peerID string + shareGeneration string + displayName *string + viewerPresence bool + roomSession bool } // outbound is one queue item: a text frame or the close frame that ends the @@ -260,13 +259,6 @@ func (s *Server) heartbeat() []*session { // send drops unless open, terminates above the buffer bound, // and otherwise queues the encoded message. func (s *Server) send(sess *session, message protocol.ServerMessage) { - // Progress is optional feedback, not route authority. Older pages reject - // totals above three; omit it for that recipient without changing the route. - if prepare, ok := message.(protocol.RouteUpdatePrepareMessage); ok && - (sess.authenticated == nil || !sess.authenticated.connectionAttemptProgress4) { - prepare.Candidate.ConnectionAttempt = nil - message = prepare - } encoded, err := protocol.EncodeServerMessage(message) if err != nil { // JSON.stringify of a server literal cannot fail; neither can this. diff --git a/src/client/App.tsx b/src/client/App.tsx index 254f64b82..4b7585145 100644 --- a/src/client/App.tsx +++ b/src/client/App.tsx @@ -11,7 +11,6 @@ import { ApiError, authenticateSiteAccess, getRuntimeCapabilities, - getOptionalRuntimeCapabilities, getSiteAccess, type SiteAccessStatus, } from "./lib/api"; @@ -202,20 +201,9 @@ function RouteLoader() { ); } -// Invitation and room-code entry share discovery without making an invitation -// pass through site authentication. Resolve before mounting media: changing the -// negotiated flag later would restart an already healthy Viewer session. function ViewerEntry({ route }: { route: ViewerRoute }) { - const [progressSupported, setProgressSupported] = useState(null); - useEffect(() => { - let active = true; - void getOptionalRuntimeCapabilities().then(capabilities => { - if (active) setProgressSupported(capabilities?.connectionAttemptProgress4 === true); - }); - return () => { active = false; }; - }, []); - return progressSupported === null ? : ( - ); } @@ -403,7 +391,6 @@ function SiteAccessGate({ sfuAvailable={capabilities?.sfu === true} sfuOnly={capabilities?.sfuOnly === true} natPredictionAvailable={capabilities?.natPrediction === true} - connectionAttemptProgress4={capabilities?.connectionAttemptProgress4 === true} onAuthorizationRequired={() => setAccess({ kind: "required", error: t("gate.expired") }) } diff --git a/src/client/components/living/QualityPresets.tsx b/src/client/components/living/QualityPresets.tsx index 289147fda..0d0302231 100644 --- a/src/client/components/living/QualityPresets.tsx +++ b/src/client/components/living/QualityPresets.tsx @@ -2,7 +2,8 @@ import { Fragment, useRef } from "react"; import { MIN_AUDIO_BITRATE } from "../../../shared/protocol"; import { QUALITY_PROFILES, QUALITY_PROFILE_KEYS, QUALITY_RESOLUTIONS, qualityResolutionEqual, - captureFramerateCeiling, VIDEO_CAPTURE_LIMITS, type VideoCaptureLimits, type QualityProfileId, type QualityResolution, + captureFramerateCeiling, supportsCaptureProfile, MIN_VIDEO_FRAMERATE, + VIDEO_CAPTURE_LIMITS, type VideoCaptureLimits, type QualityProfileId, type QualityResolution, SCREEN_AUDIO_BITRATES, type ScreenAudioPreset, } from "../../media/quality"; import { useCopy, type CopyKey } from "../../ui/copy"; @@ -27,7 +28,7 @@ export function QualityPresets({ selected, limits = VIDEO_CAPTURE_LIMITS, disabl text={vis ? undefined : t("host.quality.title", { label: caption, mbps: (profile.maxBitrate / 1_000_000).toFixed(0) })}> @@ -56,7 +57,7 @@ export function ResolutionControl({ value, limits, disabled, onChange }: { return; } const next = { width: width.valueAsNumber, height: height.valueAsNumber }; - if (captureFramerateCeiling(next, limits) < 15) { + if (captureFramerateCeiling(next, limits) < MIN_VIDEO_FRAMERATE) { reset(container); return; } @@ -76,7 +77,7 @@ export function ResolutionControl({ value, limits, disabled, onChange }: {
{Object.entries(QUALITY_RESOLUTIONS).map(([label, resolution]) => { // Selecting even the current preset replaces an unsubmitted draft. reset(controls.current!); diff --git a/src/client/lib/api.ts b/src/client/lib/api.ts index 382ba1c67..1303dff99 100644 --- a/src/client/lib/api.ts +++ b/src/client/lib/api.ts @@ -99,20 +99,6 @@ export async function getRuntimeCapabilities(signal?: AbortSignal): Promise { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), 2_000); - try { - return await getRuntimeCapabilities(controller.signal); - } catch { - return null; - } finally { - clearTimeout(timer); - } -} - export async function authenticateSiteAccess( password: string, ): Promise { diff --git a/src/client/lib/quality-preference.ts b/src/client/lib/quality-preference.ts index a5b25e8c8..db4736f10 100644 --- a/src/client/lib/quality-preference.ts +++ b/src/client/lib/quality-preference.ts @@ -3,6 +3,7 @@ import { qualitySettingsSchema, type QualitySettings, } from "../../shared/protocol"; +import { supportsCaptureProfile } from "../media/quality"; const STORAGE_KEY = "piik:quality-preference:v1"; @@ -10,7 +11,7 @@ export function readPreferredQuality(): QualitySettings { try { const stored = window.localStorage.getItem(STORAGE_KEY); const result = qualitySettingsSchema.safeParse(stored ? JSON.parse(stored) : null); - if (result.success) return result.data; + if (result.success && supportsCaptureProfile(result.data)) return result.data; } catch { // Storage may be unavailable; sharing still works with session settings. } diff --git a/src/client/media/quality.ts b/src/client/media/quality.ts index 6482c1840..779ac5722 100644 --- a/src/client/media/quality.ts +++ b/src/client/media/quality.ts @@ -2,6 +2,7 @@ import { DEFAULT_QUALITY_SETTINGS, MIN_AUDIO_BITRATE, MAX_AUDIO_BITRATE, + qualitySettingsSchema, type DegradationPreference, type QualityResolution, type QualitySettings, @@ -22,10 +23,14 @@ export type { export type QualityProfile = QualitySettings; export const STARTUP_VIDEO_ENCODED_FRAMES = 5; +export const MIN_VIDEO_FRAMERATE = 15; +export const MIN_VIDEO_BITRATE = 2_000_000; export interface VideoCaptureLimits { maxVideoResolution: QualityResolution; maxVideoPixels: number; maxVideoMacroblocksPerSecond: number; + maxVideoFramerate: number; + maxVideoBitrate: number; } // Current output boundary, independent of descriptive wire bounds and UI presets. @@ -34,6 +39,8 @@ export const VIDEO_CAPTURE_LIMITS: VideoCaptureLimits = { maxVideoResolution: { width: 3840, height: 3840 }, maxVideoPixels: 3840 * 2160, maxVideoMacroblocksPerSecond: 983_040, + maxVideoFramerate: 60, + maxVideoBitrate: 12_000_000, }; export function videoCaptureLimits(advertised?: Partial): VideoCaptureLimits { @@ -46,6 +53,8 @@ export function videoCaptureLimits(advertised?: Partial): Vi maxVideoPixels: Math.min(local.maxVideoPixels, advertised?.maxVideoPixels ?? local.maxVideoPixels), maxVideoMacroblocksPerSecond: Math.min(local.maxVideoMacroblocksPerSecond, advertised?.maxVideoMacroblocksPerSecond ?? local.maxVideoMacroblocksPerSecond), + maxVideoFramerate: Math.min(local.maxVideoFramerate, advertised?.maxVideoFramerate ?? local.maxVideoFramerate), + maxVideoBitrate: Math.min(local.maxVideoBitrate, advertised?.maxVideoBitrate ?? local.maxVideoBitrate), }; } @@ -53,10 +62,23 @@ export function captureFramerateCeiling(size: QualityResolution, limits = VIDEO_ if (![size.width, size.height].every(value => Number.isInteger(value) && value >= 4 && value % 2 === 0) || size.width > limits.maxVideoResolution.width || size.height > limits.maxVideoResolution.height || size.width * size.height > limits.maxVideoPixels) return 0; - return Math.min(60, Math.floor(limits.maxVideoMacroblocksPerSecond / + return Math.floor(Math.min(limits.maxVideoFramerate, limits.maxVideoMacroblocksPerSecond / (Math.ceil(size.width / 16) * Math.ceil(size.height / 16)))); } +export function supportsCaptureProfile(profile: QualityProfile, limits = VIDEO_CAPTURE_LIMITS, + maxAudioBitrate = MAX_SCREEN_AUDIO_BITRATE): boolean { + return qualitySettingsSchema.safeParse(profile).success && + Number.isInteger(profile.maxFramerate) && profile.maxFramerate >= MIN_VIDEO_FRAMERATE && + profile.maxFramerate <= captureFramerateCeiling(profile.resolution, limits) && + profile.maxBitrate >= MIN_VIDEO_BITRATE && profile.maxBitrate <= limits.maxVideoBitrate && + screenAudioBitrate(profile.maxAudioBitrate) <= maxAudioBitrate; +} + +function requireCaptureProfile(profile: QualityProfile): void { + if (!supportsCaptureProfile(profile)) throw new Error("Capture quality is not supported by this implementation"); +} + export const QUALITY_PROFILES = { "720p30": { resolution: { width: 1280, height: 720 }, @@ -211,6 +233,7 @@ export function qualitySettingsLabel(settings: QualitySettings): string { } function captureConstraints(profile: QualityProfile): MediaTrackConstraints { + requireCaptureProfile(profile); const resolution = profile.resolution; return { // A ceiling must not ask the Browser to fill both axes by cropping a source. @@ -261,6 +284,7 @@ export type BrowserCaptureSource = "browser" | "camera"; export async function captureBrowserSource(profile: QualityProfile, source: BrowserCaptureSource, deviceId = ""): Promise { if (source !== "camera") return captureDisplay(profile); + requireCaptureProfile(profile); const stream = await navigator.mediaDevices.getUserMedia({ audio: false, video: cameraCaptureConstraints(profile.resolution, profile.maxFramerate, deviceId) }); const track = stream.getVideoTracks()[0]; diff --git a/src/client/media/sender-quality-evidence.ts b/src/client/media/sender-quality-evidence.ts index 688fff3bf..2a6b60ffc 100644 --- a/src/client/media/sender-quality-evidence.ts +++ b/src/client/media/sender-quality-evidence.ts @@ -5,6 +5,7 @@ import { } from "../../shared/protocol"; import type { PeerSnapshot } from "../types"; import type { ConnectionMetrics } from "../types"; +import { boundedNumber, boundedInteger } from "../../shared/quality-evidence"; let senderEvidenceGeneration = 0; const connectionGenerations = new Map(); @@ -50,8 +51,10 @@ function nativeQualityState( if ( (metrics.nativeEdgeQualityState !== "healthy" && metrics.nativeEdgeQualityState !== "degraded") || - metrics.sampleWindowMs === null || - metrics.sampleTimestampMs === null || + metrics.sampleWindowMs === null || !Number.isFinite(metrics.sampleWindowMs) || + metrics.sampleWindowMs < 1_000 || metrics.sampleWindowMs > 5_000 || + boundedNumber(metrics.sampleTimestampMs) === null || + boundedInteger(frames) === null || frames === null || frames <= 0 ) { @@ -64,6 +67,22 @@ function nativeQualityState( return metrics.nativeEdgeQualityState; } +function senderDiagnostics(metrics: ConnectionMetrics, state: "unknown" | "healthy" | "degraded") { + return { + natTraversalPath: metrics.natTraversalPath, + reason: state === "unknown" ? null : metrics.qualityLimitationReason, + framesPerSecond: boundedNumber(metrics.framesPerSecond), + bitrateKbps: boundedNumber(metrics.bitrateKbps), + captureFramesPerSecond: boundedNumber(metrics.captureFramesPerSecond), + mediaSourceFramesPerSecond: boundedNumber(metrics.mediaSourceFramesPerSecond), + width: boundedInteger(metrics.frameWidth), + height: boundedInteger(metrics.frameHeight), + availableOutgoingKbps: boundedNumber(metrics.availableOutgoingKbps), + rttMs: boundedNumber(metrics.rttMs), + packetLossPercent: boundedNumber(metrics.packetLossPercent, 100), + }; +} + export function senderQualityEvidenceFromSnapshot( snapshot: PeerSnapshot, routeRevision: number, @@ -99,20 +118,7 @@ export function senderQualityEvidenceFromSnapshot( sampleTimestampMs, routeRevision, state, - diagnostics: { - natTraversalPath: snapshot.metrics.natTraversalPath, - reason: - state === "unknown" ? null : snapshot.metrics.qualityLimitationReason, - framesPerSecond: snapshot.metrics.framesPerSecond, - bitrateKbps: snapshot.metrics.bitrateKbps, - captureFramesPerSecond: snapshot.metrics.captureFramesPerSecond, - mediaSourceFramesPerSecond: snapshot.metrics.mediaSourceFramesPerSecond, - width: snapshot.metrics.frameWidth, - height: snapshot.metrics.frameHeight, - availableOutgoingKbps: snapshot.metrics.availableOutgoingKbps, - rttMs: snapshot.metrics.rttMs, - packetLossPercent: snapshot.metrics.packetLossPercent, - }, + diagnostics: senderDiagnostics(snapshot.metrics, state), }); if (!parsed.success) { return null; @@ -121,7 +127,7 @@ export function senderQualityEvidenceFromSnapshot( if (!emitUnknownOnce(unknownConnections, snapshot.connectionId)) { return null; } - if (rawSampleTimestampMs !== null) { + if (rawSampleTimestampMs !== null && boundedNumber(rawSampleTimestampMs) !== null) { connectionSampleTimestamps.set( snapshot.connectionId, rawSampleTimestampMs, @@ -163,19 +169,9 @@ export function sfuPublisherQualityEvidenceFromMetrics( state, sampleTimestampMs, diagnostics: { - natTraversalPath: metrics.natTraversalPath, - reason: state === "unknown" ? null : metrics.qualityLimitationReason, - framesPerSecond: metrics.framesPerSecond, - bitrateKbps: metrics.bitrateKbps, - captureFramesPerSecond: metrics.captureFramesPerSecond, - mediaSourceFramesPerSecond: metrics.mediaSourceFramesPerSecond, - width: metrics.frameWidth, - height: metrics.frameHeight, - videoEncodingCount: metrics.videoEncodingCount, - activeVideoEncodingCount: metrics.activeVideoEncodingCount, - availableOutgoingKbps: metrics.availableOutgoingKbps, - rttMs: metrics.rttMs, - packetLossPercent: metrics.packetLossPercent, + ...senderDiagnostics(metrics, state), + videoEncodingCount: boundedInteger(metrics.videoEncodingCount), + activeVideoEncodingCount: boundedInteger(metrics.activeVideoEncodingCount), }, }); if (!parsed.success) { @@ -185,7 +181,7 @@ export function sfuPublisherQualityEvidenceFromMetrics( if (!emitUnknownOnce(unknownPublications, publicationGeneration)) { return null; } - if (rawSampleTimestampMs !== null) { + if (rawSampleTimestampMs !== null && boundedNumber(rawSampleTimestampMs) !== null) { publicationSampleTimestamps.set( publicationGeneration, rawSampleTimestampMs, diff --git a/src/client/media/viewer-quality-evidence.ts b/src/client/media/viewer-quality-evidence.ts index 0419dd735..d2e5b3241 100644 --- a/src/client/media/viewer-quality-evidence.ts +++ b/src/client/media/viewer-quality-evidence.ts @@ -11,6 +11,7 @@ import { type ViewerQualityEvidenceMetrics, } from "../../shared/protocol"; import { packetLossPercentFromDeltas } from "../../shared/packet-loss"; +import { boundedNumber, boundedInteger, boundedSignedNumber } from "../../shared/quality-evidence"; import { EMPTY_METRICS, type ConnectionMetrics, @@ -41,29 +42,6 @@ type ViewerQualityEvidenceWindow = Pick< "windowMs" | "metrics" >; -function boundedNumber(value: number | null, maximum: number): number | null { - return value !== null && Number.isFinite(value) && value >= 0 && value <= maximum - ? value - : null; -} - -function boundedSignedNumber( - value: number | null, - absoluteMaximum: number, -): number | null { - return value !== null && - Number.isFinite(value) && - Math.abs(value) <= absoluteMaximum - ? value - : null; -} - -function boundedInteger(value: number | null, maximum: number): number | null { - return value !== null && Number.isInteger(value) && value >= 0 && value <= maximum - ? value - : null; -} - function boundedString( value: string | null, maximumLength: number, @@ -87,8 +65,8 @@ export function qualityEvidenceWindowFromMetrics( return null; } - const boundedWidth = boundedInteger(metrics.frameWidth, 16_384); - const boundedHeight = boundedInteger(metrics.frameHeight, 16_384); + const boundedWidth = boundedInteger(metrics.frameWidth); + const boundedHeight = boundedInteger(metrics.frameHeight); const width = boundedWidth !== null && boundedWidth >= 1 ? boundedWidth : null; const height = boundedHeight !== null && boundedHeight >= 1 ? boundedHeight : null; @@ -99,40 +77,19 @@ export function qualityEvidenceWindowFromMetrics( const candidate: ViewerQualityEvidenceMetrics = { natTraversalPath: metrics.natTraversalPath, ...dimensions, - framesPerSecond: boundedNumber(metrics.framesPerSecond, 240), - bitrateKbps: boundedNumber(metrics.bitrateKbps, 100_000), - packetsReceivedDelta: boundedInteger( - metrics.intervalPacketsReceived, - 1_000_000, - ), - packetsLostDelta: boundedInteger(metrics.intervalPacketsLost, 1_000_000), - rttMs: boundedNumber(metrics.rttMs, 60_000), - jitterMs: boundedNumber(metrics.jitterMs, 60_000), - framesDecodedDelta: boundedInteger( - metrics.intervalFramesDecoded, - 10_000, - ), - framesDroppedDelta: boundedInteger( - metrics.intervalFramesDropped, - 10_000, - ), - decodeMsPerFrame: boundedNumber(metrics.intervalDecodeMs, 60_000), - freezeCountDelta: boundedInteger( - metrics.intervalFreezeCount, - Number.MAX_SAFE_INTEGER, - ), - freezeDurationMsDelta: boundedNumber( - metrics.intervalFreezeDurationMs, - Number.MAX_SAFE_INTEGER, - ), - pauseCountDelta: boundedInteger( - metrics.intervalPauseCount, - Number.MAX_SAFE_INTEGER, - ), - pauseDurationMsDelta: boundedNumber( - metrics.intervalPauseDurationMs, - Number.MAX_SAFE_INTEGER, - ), + framesPerSecond: boundedNumber(metrics.framesPerSecond), + bitrateKbps: boundedNumber(metrics.bitrateKbps), + packetsReceivedDelta: boundedInteger(metrics.intervalPacketsReceived), + packetsLostDelta: boundedInteger(metrics.intervalPacketsLost), + rttMs: boundedNumber(metrics.rttMs), + jitterMs: boundedNumber(metrics.jitterMs), + framesDecodedDelta: boundedInteger(metrics.intervalFramesDecoded), + framesDroppedDelta: boundedInteger(metrics.intervalFramesDropped), + decodeMsPerFrame: boundedNumber(metrics.intervalDecodeMs), + freezeCountDelta: boundedInteger(metrics.intervalFreezeCount), + freezeDurationMsDelta: boundedNumber(metrics.intervalFreezeDurationMs), + pauseCountDelta: boundedInteger(metrics.intervalPauseCount), + pauseDurationMsDelta: boundedNumber(metrics.intervalPauseDurationMs), codec: boundedString( metrics.codec, 64, @@ -148,32 +105,20 @@ export function qualityEvidenceWindowFromMetrics( 128, /^[a-z0-9-]+=[a-z0-9]+(?:; [a-z0-9-]+=[a-z0-9]+)*$/, ), - audioBitrateKbps: boundedNumber(metrics.audioBitrateKbps, 10_000), + audioBitrateKbps: boundedNumber(metrics.audioBitrateKbps), audioPacketLossPercent: boundedNumber( metrics.audioPacketLossPercent, 100, ), - audioJitterMs: boundedNumber(metrics.audioJitterMs, 60_000), - audioVideoPlayoutDeltaMs: boundedSignedNumber( - metrics.audioVideoPlayoutDeltaMs, - 60_000, - ), - videoJitterBufferDelayMs: boundedNumber( - metrics.videoJitterBufferDelayMs, - 60_000, - ), - audioJitterBufferDelayMs: boundedNumber( - metrics.audioJitterBufferDelayMs, - 60_000, - ), + audioJitterMs: boundedNumber(metrics.audioJitterMs), + audioVideoPlayoutDeltaMs: boundedSignedNumber(metrics.audioVideoPlayoutDeltaMs), + videoJitterBufferDelayMs: boundedNumber(metrics.videoJitterBufferDelayMs), + audioJitterBufferDelayMs: boundedNumber(metrics.audioJitterBufferDelayMs), audioConcealedSamplesPercent: boundedNumber( metrics.audioConcealedSamplesPercent, 100, ), - audioConcealmentEventsDelta: boundedInteger( - metrics.intervalAudioConcealmentEvents, - 10_000, - ), + audioConcealmentEventsDelta: boundedInteger(metrics.intervalAudioConcealmentEvents), audioCodec: boundedString( metrics.audioCodec, 64, diff --git a/src/client/native/client.ts b/src/client/native/client.ts index 81e04d8c5..00ccca279 100644 --- a/src/client/native/client.ts +++ b/src/client/native/client.ts @@ -756,6 +756,13 @@ export class NativeClient { } const event = nativeEventSchema.safeParse(value); if (!event.success) { + // A rejected observation has no control authority. Drop it without + // retiring healthy media; its identity/state must still pass to be used. + if (value && typeof value === "object" && "type" in value && + (value.type === "edge-quality" || value.type === "publication-quality")) { + debugError("native", "quality-sample-invalid", event.error); + return; + } debugError("native", "protocol-failed", event.error, { stage: "event-invalid" }); this.failConnection(); return; diff --git a/src/client/native/wire.ts b/src/client/native/wire.ts index b91cb0c5d..be0c982d9 100644 --- a/src/client/native/wire.ts +++ b/src/client/native/wire.ts @@ -1,5 +1,6 @@ import { z } from "zod"; -import { sfuMediaSchema, maxAudioBitrateSchema, qualityResolutionSchema } from "../../shared/protocol"; +import { sfuMediaSchema, positiveUint32Schema, qualityResolutionSchema, + nullableSafeEvidenceNumber, nullableSafeEvidenceInteger } from "../../shared/protocol"; export const NATIVE_CLIENT_PROTOCOL = 10; export const NATIVE_CLIENT_PORT_START = 39_721; @@ -33,10 +34,12 @@ export const nativeHealthSchema = nativeDiscoveryIdentitySchema.extend({ .regex(/^[A-Za-z0-9_-]+$/), nativeMedia: z .object({ - maxAudioBitrate: maxAudioBitrateSchema.optional(), + maxAudioBitrate: positiveUint32Schema.optional(), maxVideoResolution: qualityResolutionSchema.optional(), - maxVideoPixels: z.number().int().min(16).max(8192 * 8192).optional(), + maxVideoPixels: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).optional(), maxVideoMacroblocksPerSecond: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).optional(), + maxVideoFramerate: z.number().positive().max(Number.MAX_SAFE_INTEGER).optional(), + maxVideoBitrate: positiveUint32Schema.optional(), receiverReuse: z.boolean().default(false), video: z.boolean().default(false), processAudio: z.boolean().default(false), @@ -270,25 +273,20 @@ export const nativeEventSchema = z.discriminatedUnion("type", [ rtpStatsId: z.string().min(1).max(256), trackIdentifier: z.string().min(1).max(256), codec: nativeVideoCodecSchema, - videoEncodingCount: z.number().int().min(1).max(3), - activeVideoEncodingCount: z.number().int().min(0).max(3), + videoEncodingCount: nullableSafeEvidenceInteger.catch(null), + activeVideoEncodingCount: nullableSafeEvidenceInteger.catch(null), rid: z.string().max(16), intervalFramesSent: z .number() .int() .nonnegative() .max(Number.MAX_SAFE_INTEGER), - framesPerSecond: z.number().finite().nonnegative().max(240), - width: z.number().int().nonnegative().max(16_384), - height: z.number().int().nonnegative().max(16_384), - bitrateKbps: z.number().finite().nonnegative().max(100_000), - audioBitrateKbps: z.number().finite().nonnegative().max(1_000), - availableOutgoingKbps: z - .number() - .finite() - .nonnegative() - .max(100_000) - .nullable(), + framesPerSecond: nullableSafeEvidenceNumber.catch(null), + width: nullableSafeEvidenceInteger.catch(null), + height: nullableSafeEvidenceInteger.catch(null), + bitrateKbps: nullableSafeEvidenceNumber.catch(null), + audioBitrateKbps: nullableSafeEvidenceNumber.catch(null), + availableOutgoingKbps: nullableSafeEvidenceNumber.catch(null), state: z.enum(["unknown", "healthy", "degraded"]), reason: z.enum(["none", "bandwidth"]).nullable(), }) @@ -385,11 +383,11 @@ export const nativeEventSchema = z.discriminatedUnion("type", [ .int() .nonnegative() .max(Number.MAX_SAFE_INTEGER), - framesPerSecond: z.number().finite().nonnegative().max(240), - bitrateKbps: z.number().finite().nonnegative().max(100_000), - availableOutgoingKbps: z.number().finite().nonnegative().max(100_000), - width: z.number().int().nonnegative().max(16_384), - height: z.number().int().nonnegative().max(16_384), + framesPerSecond: nullableSafeEvidenceNumber.catch(null), + bitrateKbps: nullableSafeEvidenceNumber.catch(null), + availableOutgoingKbps: nullableSafeEvidenceNumber.catch(null), + width: nullableSafeEvidenceInteger.catch(null), + height: nullableSafeEvidenceInteger.catch(null), }) .strict() .superRefine((event, context) => { diff --git a/src/client/pages/HostPage.tsx b/src/client/pages/HostPage.tsx index a305eaccd..900d69580 100644 --- a/src/client/pages/HostPage.tsx +++ b/src/client/pages/HostPage.tsx @@ -129,6 +129,9 @@ import { DEGRADATION_PREFERENCE_KEYS, videoCaptureLimits, captureFramerateCeiling, + supportsCaptureProfile, + MIN_VIDEO_FRAMERATE, + MIN_VIDEO_BITRATE, qualityResolutionEqual, qualitySettingsEqual, qualitySettingsLabel, @@ -342,7 +345,6 @@ interface HostPageProps { sfuAvailable?: boolean; sfuOnly?: boolean; natPredictionAvailable?: boolean; - connectionAttemptProgress4?: boolean; launchedByClient?: boolean; onAuthorizationRequired?: () => void; } @@ -365,7 +367,6 @@ export function HostPage({ sfuAvailable = false, sfuOnly = false, natPredictionAvailable = false, - connectionAttemptProgress4 = false, launchedByClient = false, onAuthorizationRequired, }: HostPageProps = {}) { @@ -1631,7 +1632,7 @@ export function HostPage({ pendingAppQualityPreferenceRef.current = null; if (pending) preference.save(pending); const saved = await preference.read(); - if (!signal.aborted && saved && !qualityPreferenceEditedRef.current && + if (!signal.aborted && saved && supportsCaptureProfile(saved) && !qualityPreferenceEditedRef.current && qualitySettingsRef.current === initial && activeGenerationRef.current === null && roomMutationRef.current === null) { commitQuality(saved); @@ -1676,6 +1677,12 @@ export function HostPage({ } async function changeQuality(nextProfile: QualitySettings): Promise { + if (!supportsCaptureProfile(nextProfile, videoLimits, audioBitrateCeiling)) { + advancedQualityRef.current = qualitySettingsRef.current; + setAdvancedQuality(qualitySettingsRef.current); + setNoticeError(new Error("Capture quality is not supported by this implementation"), "quality"); + return; + } qualityPreferenceEditedRef.current = true; advancedQualityRef.current = nextProfile; setAdvancedQuality(nextProfile); @@ -2516,7 +2523,6 @@ export function HostPage({ qualitySettings: qualitySettingsRef.current, routePolicy: routePolicyRef.current, viewerPresence: true, - ...(connectionAttemptProgress4 ? { connectionAttemptProgress4: true } : {}), displayName: initialDisplayName, }, { @@ -3590,7 +3596,7 @@ export function HostPage({ ? nativeClientRef.current?.health.nativeMedia.maxAudioBitrate ?? MAX_SCREEN_AUDIO_BITRATE : MAX_SCREEN_AUDIO_BITRATE); const videoLimits = videoCaptureLimits(nativeActive ? nativeClientRef.current?.health.nativeMedia : undefined); - const framerateCeiling = Math.max(15, captureFramerateCeiling(advancedQuality.resolution, videoLimits)); + const framerateCeiling = captureFramerateCeiling(advancedQuality.resolution, videoLimits); return (
@@ -3856,11 +3862,11 @@ export function HostPage({ changeAdvancedQuality({ @@ -3883,11 +3889,11 @@ export function HostPage({ changeAdvancedQuality({ diff --git a/src/client/pages/ViewerPage.tsx b/src/client/pages/ViewerPage.tsx index da1177c65..1339113da 100644 --- a/src/client/pages/ViewerPage.tsx +++ b/src/client/pages/ViewerPage.tsx @@ -130,7 +130,6 @@ interface ViewerPageProps { viewerGrant?: string; invalidGrant?: boolean; launchedByClient?: boolean; - connectionAttemptProgress4?: boolean; } type ViewerQualityEvidence = Extract< @@ -170,7 +169,6 @@ export function ViewerPage({ viewerGrant, invalidGrant = false, launchedByClient = false, - connectionAttemptProgress4 = false, }: ViewerPageProps) { const { lang, t, vis, titleFrames } = useCopy(); const viewerClientId = useMemo( @@ -576,7 +574,6 @@ export function ViewerPage({ : {}), displayName: displayNameRef.current, viewerPresence: true, - ...(connectionAttemptProgress4 ? { connectionAttemptProgress4: true } : {}), }, { onStatus: (status) => { @@ -1844,7 +1841,7 @@ export function ViewerPage({ viewerRelaySourceKey = null; releaseNativeClient(); }; - }, [roomId, viewerGrant, viewerPasswordAttempt, launchedByClient, connectionAttemptProgress4]); + }, [roomId, viewerGrant, viewerPasswordAttempt, launchedByClient]); useEffect(() => { const video = videoRef.current; diff --git a/src/shared/protocol.ts b/src/shared/protocol.ts index 0d5b374bd..b11e6dd2f 100644 --- a/src/shared/protocol.ts +++ b/src/shared/protocol.ts @@ -174,10 +174,12 @@ export const roleSchema = z.enum(["host", "viewer"]); export type Role = z.infer; // Describes a requested ceiling, not encoder availability or an allocation. -// Match the existing media-description bound; adapters retain their own limits. +// WebRTC uses unsigned-long dimensions/bitrates and a double frame rate. +// Alignment, usable ranges and allocation budgets belong to the media adapter. +export const positiveUint32Schema = z.number().int().positive().max(0xffff_ffff); export const qualityResolutionSchema = z.object({ - width: z.number().int().min(4).max(8192).multipleOf(2), - height: z.number().int().min(4).max(8192).multipleOf(2), + width: positiveUint32Schema, + height: positiveUint32Schema, }).strict(); export type QualityResolution = z.infer; @@ -197,8 +199,8 @@ export const maxAudioBitrateSchema = z.number().int().min(MIN_AUDIO_BITRATE).max export const qualitySettingsSchema = z .object({ resolution: qualityResolutionSchema, - maxFramerate: z.number().int().min(15).max(60), - maxBitrate: z.number().int().min(2_000_000).max(12_000_000), + maxFramerate: z.number().positive().max(Number.MAX_SAFE_INTEGER), + maxBitrate: positiveUint32Schema, degradationPreference: degradationPreferenceSchema, maxAudioBitrate: maxAudioBitrateSchema.optional(), }) @@ -227,7 +229,6 @@ export const DEFAULT_ROUTE_POLICY = { } as const satisfies RoutePolicy; export const runtimeCapabilitiesSchema = z.object({ - connectionAttemptProgress4: z.boolean().optional(), sfu: z.boolean().default(false), sfuOnly: z.boolean().optional(), natPrediction: z.boolean().default(false), @@ -345,12 +346,12 @@ export const sfuMediaSchema = z.object({ codec: z.enum(["h264", "vp8"]), layers: z.array(z.object({ rid: z.string().max(16), - width: z.number().int().min(1).max(8192), - height: z.number().int().min(1).max(8192), - bitrate: z.number().int().min(1).max(100_000_000), + width: positiveUint32Schema, + height: positiveUint32Schema, + bitrate: positiveUint32Schema, }).strict()).min(1).max(3), audio: z.boolean(), - audioBitrate: z.number().int().min(0).max(510_000), + audioBitrate: z.number().int().min(0).max(MAX_AUDIO_BITRATE), }).strict().refine((media) => (!media.audio || media.audioBitrate > 0) && new Set(media.layers.map((layer) => layer.rid)).size === media.layers.length && @@ -385,8 +386,8 @@ export const preparedRouteCandidateSchema = z transport: z.enum(["direct", "sfu"]), qualityProbe: z.boolean(), connectionAttempt: z.object({ - current: z.number().int().min(1).max(4), - total: z.union([z.literal(3), z.literal(4)]), + current: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + total: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), }).strict().refine(({ current, total }) => current <= total).optional(), }) .strict(); @@ -602,17 +603,14 @@ const nullableEvidenceNumber = (maximum: number) => const nullableSignedEvidenceNumber = (absoluteMaximum: number) => z.number().finite().min(-absoluteMaximum).max(absoluteMaximum).nullable(); -const nullableEvidenceInteger = (maximum: number) => - z.number().int().min(0).max(maximum).nullable(); - -const nullableSafeEvidenceNumber = z +export const nullableSafeEvidenceNumber = z .number() .finite() .min(0) .max(Number.MAX_SAFE_INTEGER) .nullable(); -const nullableSafeEvidenceInteger = z +export const nullableSafeEvidenceInteger = z .number() .int() .min(0) @@ -622,17 +620,17 @@ const nullableSafeEvidenceInteger = z export const viewerQualityEvidenceMetricsSchema = z .object({ natTraversalPath: z.enum(NAT_TRAVERSAL_PATHS), - width: z.number().int().min(1).max(16_384).nullable(), - height: z.number().int().min(1).max(16_384).nullable(), - framesPerSecond: nullableEvidenceNumber(240), - bitrateKbps: nullableEvidenceNumber(100_000), - packetsReceivedDelta: nullableEvidenceInteger(1_000_000), - packetsLostDelta: nullableEvidenceInteger(1_000_000), - rttMs: nullableEvidenceNumber(60_000), - jitterMs: nullableEvidenceNumber(60_000), - framesDecodedDelta: nullableEvidenceInteger(10_000), - framesDroppedDelta: nullableEvidenceInteger(10_000), - decodeMsPerFrame: nullableEvidenceNumber(60_000), + width: z.number().int().min(1).max(Number.MAX_SAFE_INTEGER).nullable(), + height: z.number().int().min(1).max(Number.MAX_SAFE_INTEGER).nullable(), + framesPerSecond: nullableSafeEvidenceNumber, + bitrateKbps: nullableSafeEvidenceNumber, + packetsReceivedDelta: nullableSafeEvidenceInteger, + packetsLostDelta: nullableSafeEvidenceInteger, + rttMs: nullableSafeEvidenceNumber, + jitterMs: nullableSafeEvidenceNumber, + framesDecodedDelta: nullableSafeEvidenceInteger, + framesDroppedDelta: nullableSafeEvidenceInteger, + decodeMsPerFrame: nullableSafeEvidenceNumber, freezeCountDelta: nullableSafeEvidenceInteger, freezeDurationMsDelta: nullableSafeEvidenceNumber, pauseCountDelta: nullableSafeEvidenceInteger, @@ -652,14 +650,14 @@ export const viewerQualityEvidenceMetricsSchema = z .max(128) .regex(/^[a-z0-9-]+=[a-z0-9]+(?:; [a-z0-9-]+=[a-z0-9]+)*$/) .nullable(), - audioBitrateKbps: nullableEvidenceNumber(10_000), + audioBitrateKbps: nullableSafeEvidenceNumber, audioPacketLossPercent: nullableEvidenceNumber(100), - audioJitterMs: nullableEvidenceNumber(60_000), - audioVideoPlayoutDeltaMs: nullableSignedEvidenceNumber(60_000), - videoJitterBufferDelayMs: nullableEvidenceNumber(60_000), - audioJitterBufferDelayMs: nullableEvidenceNumber(60_000), + audioJitterMs: nullableSafeEvidenceNumber, + audioVideoPlayoutDeltaMs: nullableSignedEvidenceNumber(Number.MAX_SAFE_INTEGER), + videoJitterBufferDelayMs: nullableSafeEvidenceNumber, + audioJitterBufferDelayMs: nullableSafeEvidenceNumber, audioConcealedSamplesPercent: nullableEvidenceNumber(100), - audioConcealmentEventsDelta: nullableEvidenceInteger(10_000), + audioConcealmentEventsDelta: nullableSafeEvidenceInteger, audioCodec: z .string() .max(64) @@ -703,20 +701,16 @@ const senderQualityDiagnosticsSchema = z .object({ natTraversalPath: z.enum(NAT_TRAVERSAL_PATHS), reason: z.enum(["none", "bandwidth", "cpu"]).nullable(), - framesPerSecond: nullableEvidenceNumber(240), - bitrateKbps: nullableEvidenceNumber(100_000), - captureFramesPerSecond: nullableEvidenceNumber(240).optional(), - mediaSourceFramesPerSecond: nullableEvidenceNumber(240).optional(), - width: nullableEvidenceInteger(16_384).optional(), - height: nullableEvidenceInteger(16_384).optional(), - videoEncodingCount: nullableEvidenceInteger( - Number.MAX_SAFE_INTEGER, - ).optional(), - activeVideoEncodingCount: nullableEvidenceInteger( - Number.MAX_SAFE_INTEGER, - ).optional(), - availableOutgoingKbps: nullableEvidenceNumber(100_000).optional(), - rttMs: nullableEvidenceNumber(60_000).optional(), + framesPerSecond: nullableSafeEvidenceNumber, + bitrateKbps: nullableSafeEvidenceNumber, + captureFramesPerSecond: nullableSafeEvidenceNumber.optional(), + mediaSourceFramesPerSecond: nullableSafeEvidenceNumber.optional(), + width: nullableSafeEvidenceInteger.optional(), + height: nullableSafeEvidenceInteger.optional(), + videoEncodingCount: nullableSafeEvidenceInteger.optional(), + activeVideoEncodingCount: nullableSafeEvidenceInteger.optional(), + availableOutgoingKbps: nullableSafeEvidenceNumber.optional(), + rttMs: nullableSafeEvidenceNumber.optional(), packetLossPercent: nullableEvidenceNumber(100).optional(), }) .strict(); @@ -831,7 +825,6 @@ const authenticateMessageSchema = z.discriminatedUnion("role", [ qualitySettings: qualitySettingsSchema.optional(), routePolicy: routePolicySchema.default(DEFAULT_ROUTE_POLICY), viewerPresence: z.literal(true).optional(), - connectionAttemptProgress4: z.literal(true).optional(), displayName: displayNameSchema.optional(), }) .strict() @@ -855,7 +848,6 @@ const authenticateMessageSchema = z.discriminatedUnion("role", [ viewerPassword: viewerPasswordSchema.optional(), displayName: displayNameSchema.optional(), viewerPresence: z.literal(true).optional(), - connectionAttemptProgress4: z.literal(true).optional(), }) .strict(), ]); diff --git a/src/shared/quality-evidence.ts b/src/shared/quality-evidence.ts new file mode 100644 index 000000000..bb7962e4a --- /dev/null +++ b/src/shared/quality-evidence.ts @@ -0,0 +1,13 @@ +// Observations do not allocate media resources. Preserve representable values; +// unavailable or invalid measurements stay null, never clamped into false facts. +export function boundedNumber(value: number | null, maximum = Number.MAX_SAFE_INTEGER): number | null { + return value !== null && Number.isFinite(value) && value >= 0 && value <= maximum ? value : null; +} + +export function boundedInteger(value: number | null, maximum = Number.MAX_SAFE_INTEGER): number | null { + return value !== null && Number.isSafeInteger(value) && value >= 0 && value <= maximum ? value : null; +} + +export function boundedSignedNumber(value: number | null, maximum = Number.MAX_SAFE_INTEGER): number | null { + return value !== null && Number.isFinite(value) && Math.abs(value) <= maximum ? value : null; +} diff --git a/tests/client-state.test.ts b/tests/client-state.test.ts index 7c9e2b1fa..0b05b1e9e 100644 --- a/tests/client-state.test.ts +++ b/tests/client-state.test.ts @@ -12,7 +12,6 @@ import { authenticateSiteAccess, createRoom, getSiteAccess, - getOptionalRuntimeCapabilities, replaceOwnedRoom, updateRoomAccess, } from "../src/client/lib/api.ts"; @@ -65,34 +64,6 @@ afterEach(() => { setCopy({ lang: "zh", vis: false }); }); -describe("optional Viewer capability discovery", () => { - it("returns supported progress and tolerates an unavailable or malformed descriptor", async () => { - for (const [body, status, expected] of [ - [{ connectionAttemptProgress4: true }, 200, true], - [{}, 200, undefined], - [{}, 503, undefined], - [{ connectionAttemptProgress4: 4 }, 200, undefined], - ] as const) { - vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(JSON.stringify(body), { status }))); - expect((await getOptionalRuntimeCapabilities())?.connectionAttemptProgress4).toBe(expected); - } - }); - - it("abandons stalled discovery without preventing viewing", async () => { - vi.useFakeTimers(); - let observed: AbortSignal | undefined; - vi.stubGlobal("fetch", vi.fn((_url: string, options: RequestInit) => new Promise((_resolve, reject) => { - observed = options.signal!; - observed.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError"))); - }))); - const discovery = getOptionalRuntimeCapabilities(); - await vi.advanceTimersByTimeAsync(2_000); - await expect(discovery).resolves.toBeNull(); - expect(observed?.aborted).toBe(true); - expect(vi.getTimerCount()).toBe(0); - }); -}); - function routeAuthenticated( shareGeneration: string | null = null, qualitySettings: QualitySettings = DEFAULT_QUALITY_SETTINGS, diff --git a/tests/fixtures/wire-samples.json b/tests/fixtures/wire-samples.json index 62ca26220..6ab72a219 100644 --- a/tests/fixtures/wire-samples.json +++ b/tests/fixtures/wire-samples.json @@ -299,7 +299,7 @@ "name": "set-quality-settings-framerate-below-floor", "schema": "client", "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":14,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", - "valid": false + "valid": true }, { "name": "set-quality-settings-fractional-bitrate", @@ -1157,25 +1157,25 @@ "name": "set-quality-settings-too-wide", "schema": "client", "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":8194,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", - "valid": false + "valid": true }, { "name": "set-quality-settings-too-tall", "schema": "client", "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":8194},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", - "valid": false + "valid": true }, { "name": "set-quality-settings-too-small", "schema": "client", "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":2,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", - "valid": false + "valid": true }, { "name": "set-quality-settings-odd-size", "schema": "client", "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1919,\"height\":1080},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", - "valid": false + "valid": true }, { "name": "set-quality-settings-fractional-size", @@ -1200,6 +1200,84 @@ "schema": "client", "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080,\"label\":\"1080p\"},\"maxFramerate\":30,\"maxBitrate\":5000000,\"degradationPreference\":\"balanced\",\"maxAudioBitrate\":128000}}", "valid": false + }, + { + "name": "set-quality-settings-higher-rate", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":120,\"maxBitrate\":40000000,\"degradationPreference\":\"maintain-resolution\"}}", + "valid": true + }, + { + "name": "set-quality-settings-fractional-framerate", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":29.97,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", + "valid": true + }, + { + "name": "set-quality-settings-zero-framerate", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":0,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", + "valid": false + }, + { + "name": "set-quality-settings-unrepresentable-bitrate", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":1920,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":4294967296,\"degradationPreference\":\"maintain-resolution\"}}", + "valid": false + }, + { + "name": "set-quality-settings-unrepresentable-size", + "schema": "client", + "json": "{\"type\":\"set-quality-settings\",\"qualitySettings\":{\"resolution\":{\"width\":4294967296,\"height\":1080},\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\"}}", + "valid": false + }, + { + "name": "sender-quality-evidence-high-observations", + "schema": "client", + "json": "{\"type\":\"sender-quality-evidence\",\"childPeerId\":\"viewer_12345678\",\"connectionId\":\"connection_12345678\",\"rtpStatsId\":\"rtp-stats-1\",\"trackIdentifier\":\"track-1\",\"sampleTimestampMs\":2000,\"routeRevision\":3,\"state\":\"degraded\",\"diagnostics\":{\"natTraversalPath\":\"ordinary\",\"reason\":\"bandwidth\",\"framesPerSecond\":480,\"bitrateKbps\":200000,\"rttMs\":70000,\"width\":32768,\"height\":32768,\"availableOutgoingKbps\":1000000}}", + "valid": true + }, + { + "name": "sender-quality-evidence-unrepresentable-observation", + "schema": "client", + "json": "{\"type\":\"sender-quality-evidence\",\"childPeerId\":\"viewer_12345678\",\"connectionId\":\"connection_12345678\",\"rtpStatsId\":\"rtp-stats-1\",\"trackIdentifier\":\"track-1\",\"sampleTimestampMs\":2000,\"routeRevision\":3,\"state\":\"degraded\",\"diagnostics\":{\"natTraversalPath\":\"ordinary\",\"reason\":\"bandwidth\",\"framesPerSecond\":480,\"bitrateKbps\":9007199254740992,\"rttMs\":70000,\"width\":32768,\"height\":32768,\"availableOutgoingKbps\":1000000}}", + "valid": false + }, + { + "name": "sfu-publisher-quality-evidence-high-observations", + "schema": "client", + "json": "{\"type\":\"sfu-publisher-quality-evidence\",\"publicationGeneration\":\"publication_generation_12345678\",\"routeRevision\":4,\"state\":\"healthy\",\"sampleTimestampMs\":2000,\"diagnostics\":{\"natTraversalPath\":\"ordinary\",\"reason\":\"none\",\"framesPerSecond\":480,\"bitrateKbps\":200000,\"captureFramesPerSecond\":30,\"mediaSourceFramesPerSecond\":null,\"width\":32768,\"height\":32768,\"videoEncodingCount\":1,\"activeVideoEncodingCount\":1,\"availableOutgoingKbps\":1000000,\"rttMs\":70000,\"packetLossPercent\":0.1}}", + "valid": true + }, + { + "name": "sfu-publisher-quality-evidence-unrepresentable-observation", + "schema": "client", + "json": "{\"type\":\"sfu-publisher-quality-evidence\",\"publicationGeneration\":\"publication_generation_12345678\",\"routeRevision\":4,\"state\":\"healthy\",\"sampleTimestampMs\":2000,\"diagnostics\":{\"natTraversalPath\":\"ordinary\",\"reason\":\"none\",\"framesPerSecond\":480,\"bitrateKbps\":9007199254740992,\"captureFramesPerSecond\":30,\"mediaSourceFramesPerSecond\":null,\"width\":32768,\"height\":32768,\"videoEncodingCount\":1,\"activeVideoEncodingCount\":1,\"availableOutgoingKbps\":1000000,\"rttMs\":70000,\"packetLossPercent\":0.1}}", + "valid": false + }, + { + "name": "viewer-quality-evidence-high-observations", + "schema": "client", + "json": "{\"type\":\"viewer-quality-evidence\",\"guard\":{\"connectionId\":\"connection_12345678\",\"routeRevision\":0,\"presentationEpoch\":0},\"sequence\":0,\"windowMs\":2000,\"metrics\":{\"natTraversalPath\":\"ordinary\",\"width\":32768,\"height\":32768,\"framesPerSecond\":480,\"bitrateKbps\":200000,\"packetsReceivedDelta\":1500,\"packetsLostDelta\":2,\"rttMs\":70000,\"jitterMs\":3.5,\"framesDecodedDelta\":120,\"framesDroppedDelta\":1,\"decodeMsPerFrame\":2.4,\"freezeCountDelta\":0,\"freezeDurationMsDelta\":0,\"pauseCountDelta\":0,\"pauseDurationMsDelta\":0,\"codec\":\"video/H264\",\"codecProfile\":\"profile-level-id=42e01f\",\"codecParameters\":\"packetization-mode=1; level-asymmetry-allowed=1\",\"audioBitrateKbps\":20000,\"audioPacketLossPercent\":0.2,\"audioJitterMs\":2.5,\"audioVideoPlayoutDeltaMs\":-70000,\"videoJitterBufferDelayMs\":24,\"audioJitterBufferDelayMs\":18,\"audioConcealedSamplesPercent\":1,\"audioConcealmentEventsDelta\":3,\"audioCodec\":\"audio/opus\"}}", + "valid": true + }, + { + "name": "viewer-quality-evidence-unrepresentable-observation", + "schema": "client", + "json": "{\"type\":\"viewer-quality-evidence\",\"guard\":{\"connectionId\":\"connection_12345678\",\"routeRevision\":0,\"presentationEpoch\":0},\"sequence\":0,\"windowMs\":2000,\"metrics\":{\"natTraversalPath\":\"ordinary\",\"width\":32768,\"height\":32768,\"framesPerSecond\":480,\"bitrateKbps\":9007199254740992,\"packetsReceivedDelta\":1500,\"packetsLostDelta\":2,\"rttMs\":70000,\"jitterMs\":3.5,\"framesDecodedDelta\":120,\"framesDroppedDelta\":1,\"decodeMsPerFrame\":2.4,\"freezeCountDelta\":0,\"freezeDurationMsDelta\":0,\"pauseCountDelta\":0,\"pauseDurationMsDelta\":0,\"codec\":\"video/H264\",\"codecProfile\":\"profile-level-id=42e01f\",\"codecParameters\":\"packetization-mode=1; level-asymmetry-allowed=1\",\"audioBitrateKbps\":20000,\"audioPacketLossPercent\":0.2,\"audioJitterMs\":2.5,\"audioVideoPlayoutDeltaMs\":-70000,\"videoJitterBufferDelayMs\":24,\"audioJitterBufferDelayMs\":18,\"audioConcealedSamplesPercent\":1,\"audioConcealmentEventsDelta\":3,\"audioCodec\":\"audio/opus\"}}", + "valid": false + }, + { + "name": "sfu-numeric-description-above-implementation", + "schema": "client", + "json": "{\"type\":\"sfu-signal\",\"revision\":9,\"publicationGeneration\":\"publication_12345678\",\"connectionId\":\"connection_12345678\",\"kind\":\"description\",\"description\":{\"type\":\"offer\",\"sdp\":\"v=0\\\\r\\\\n\"},\"media\":{\"codec\":\"vp8\",\"layers\":[{\"rid\":\"\",\"width\":32768,\"height\":32768,\"bitrate\":200000000}],\"audio\":false,\"audioBitrate\":128000}}", + "valid": true + }, + { + "name": "sfu-unrepresentable-description", + "schema": "client", + "json": "{\"type\":\"sfu-signal\",\"revision\":9,\"publicationGeneration\":\"publication_12345678\",\"connectionId\":\"connection_12345678\",\"kind\":\"description\",\"description\":{\"type\":\"offer\",\"sdp\":\"v=0\\\\r\\\\n\"},\"media\":{\"codec\":\"vp8\",\"layers\":[{\"rid\":\"\",\"width\":4294967296,\"height\":32768,\"bitrate\":200000000}],\"audio\":false,\"audioBitrate\":128000}}", + "valid": false } ], "serverMessages": [ diff --git a/tests/host-quality-ownership.test.ts b/tests/host-quality-ownership.test.ts index b800a096a..cc8f92838 100644 --- a/tests/host-quality-ownership.test.ts +++ b/tests/host-quality-ownership.test.ts @@ -4,7 +4,7 @@ import ts from "typescript"; import { describe, expect, it, vi } from "vitest"; import { QUALITY_PROFILES, qualitySettingsEqual, qualitySettingsLabel, qualityResolutionEqual, - screenAudioBitrate, videoQualitySettingsEqual, + screenAudioBitrate, videoQualitySettingsEqual, supportsCaptureProfile, videoCaptureLimits, MAX_SCREEN_AUDIO_BITRATE, type QualitySettings, } from "../src/client/media/quality"; import { NativeSenderPeer } from "../src/client/native/native-sender-peer"; @@ -109,6 +109,7 @@ function fixture(launchedByClient = true) { applyCaptureProfile: vi.fn(async (_stream: unknown, profile: QualitySettings) => { physical = profile; }), captureDetails: () => ({}), qualitySettingsEqual, qualityResolutionEqual, videoQualitySettingsEqual, screenAudioBitrate, qualitySettingsLabel, + supportsCaptureProfile, videoLimits: videoCaptureLimits(), audioBitrateCeiling: MAX_SCREEN_AUDIO_BITRATE, say: (key: string) => key, syncHostSfuQualityWarning: () => null, NativeSenderPeer, discardPreparedHostChild: vi.fn(), removePeer: vi.fn(), startPeer: vi.fn(async () => undefined), nativePreviewTailRef: ref(Promise.resolve()), setSwitchingSource: vi.fn(), @@ -225,7 +226,6 @@ describe("Host room ownership", () => { current.generationRef.current = 1; Object.assign(current.context, { SignalingClient: Signal, hostRoomSessionAvailable: true, roomInteractionsAvailable: true, - connectionAttemptProgress4:true, visRef: ref(false), hostClientIdRef: ref(null), displayNameRef: ref("Host"), getStableClientId: () => "host-client", defaultHostDisplayName: () => "Host", readDisplayName: () => "Host", setDisplayName: vi.fn(), setDisplayNameDraft: vi.fn(), setDisplayNameError: vi.fn(), @@ -426,6 +426,27 @@ describe("Host room-link copy feedback", () => { }); describe("Host quality ownership", () => { + it("rejects unsupported settings without changing capture, room or saved intent", async () => { + const current = fixture(); + await current.change({ ...original, maxFramerate: 120, maxBitrate: 40_000_000 }); + expect(current.qualitySettingsRef.current).toEqual(original); + expect(current.applyCaptureProfile).not.toHaveBeenCalled(); + expect(current.signalRef.current.setHostQualitySettings).not.toHaveBeenCalled(); + expect(current.context.savePreferredQuality).not.toHaveBeenCalled(); + expect(current.advancedQualityRef.current).toEqual(original); + }); + + it("ignores an unsupported App preference without erasing it", async () => { + const current = fixture(); + current.activeGenerationRef.current = null; + const preference = { read: vi.fn(async () => ({ ...original, maxFramerate: 120 })), save: vi.fn() }; + current.context.AppQualityPreference = { connect: vi.fn(async () => preference) }; + await current.context.loadAppQualityPreference(new AbortController().signal); + expect(current.qualitySettingsRef.current).toEqual(original); + expect(preference.save).not.toHaveBeenCalled(); + expect(current.NativeClient.connect).not.toHaveBeenCalled(); + }); + it("restores App preferences without acquiring media or changing a room", async () => { const current = fixture(); current.activeGenerationRef.current = null; diff --git a/tests/native-client.test.ts b/tests/native-client.test.ts index 180f776c5..35299606d 100644 --- a/tests/native-client.test.ts +++ b/tests/native-client.test.ts @@ -35,6 +35,10 @@ describe("native App private wire", () => { const nativeMedia = { ...health.nativeMedia, maxVideoResolution: { width: 3840, height: 3840 }, maxVideoPixels: 3840 * 2160, maxVideoMacroblocksPerSecond: 983_040 }; expect(nativeHealthSchema.parse({ ...health, nativeMedia }).nativeMedia).toMatchObject(nativeMedia); + const stronger = { ...nativeMedia, maxVideoResolution: { width: 32768, height: 32768 }, + maxVideoPixels: 32768 ** 2, maxVideoFramerate: 120, maxVideoBitrate: 100_000_000, + maxAudioBitrate: 960_000 }; + expect(nativeHealthSchema.parse({ ...health, nativeMedia: stronger }).nativeMedia).toMatchObject(stronger); expect(nativeHealthSchema.safeParse({ ...health, nativeMedia: { ...nativeMedia, maxVideoPixels: 0 } }).success).toBe(false); expect(nativeHealthSchema.safeParse({ ...health, nativeMedia: { ...nativeMedia, maxVideoMacroblocksPerSecond: -1 } }).success).toBe(false); }); @@ -808,4 +812,47 @@ describe("native App private wire", () => { }).success, ).toBe(true); }); + + it.each(["edge-quality", "publication-quality"] as const)("isolates %s observations from control lifetime", async (type) => { + class Socket extends EventTarget { + static readonly OPEN = 1; + static readonly CLOSING = 2; + static latest: Socket; + readyState = Socket.OPEN; + protocol = `piik-client-v10.${health.instanceToken}`; + close = vi.fn(() => { this.readyState = Socket.CLOSING; this.dispatchEvent(new Event("close")); }); + constructor() { super(); Socket.latest = this; queueMicrotask(() => this.dispatchEvent(new Event("open"))); } + emit(value: unknown) { this.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(value) })); } + send(payload: string) { + const request = JSON.parse(payload); + this.emit({ version: 10, id: request.id, type: request.type === "hello" ? "ready" : "pong" }); + } + } + vi.stubGlobal("WebSocket", Socket); + vi.stubGlobal("window", { setTimeout, clearTimeout }); + vi.stubGlobal("fetch", vi.fn(async () => Response.json(health))); + const client = (await NativeClient.connect())!; + const received = vi.fn(); + client.onEvent(received); + const event = { version: 10, type, shareId: "share_123456", connectionId: "connection_123456", + sampleTimestampMs: 10_000, sampleWindowMs: 2_000, rtpStatsId: "rtp-1", trackIdentifier: "screen", + state: "healthy", reason: "none", framesPerSecond: 480, bitrateKbps: 120_000, + availableOutgoingKbps: 1_000_000, width: 1920, height: 1080, + ...(type === "edge-quality" ? { intervalFramesEncoded: 60 } : { + publicationGeneration: "publication_123456", codec: "vp8", videoEncodingCount: 2, + activeVideoEncodingCount: 2, rid: "h", intervalFramesSent: 60, audioBitrateKbps: 192, + }), + }; + Socket.latest.emit(event); + expect(received).toHaveBeenLastCalledWith(event); + Socket.latest.emit({ ...event, framesPerSecond: -1, availableOutgoingKbps: "invalid" }); + expect(received).toHaveBeenLastCalledWith({ ...event, framesPerSecond: null, availableOutgoingKbps: null }); + // Invalid identity/state never becomes evidence, but cannot retire media. + Socket.latest.emit({ ...event, shareId: "bad", state: "degraded", reason: "none" }); + expect(received).toHaveBeenCalledTimes(2); + await expect(client.ping()).resolves.toBeUndefined(); + expect(Socket.latest.close).not.toHaveBeenCalled(); + Socket.latest.emit({ version: 10, type: "edge-state", shareId: "bad" }); + expect(Socket.latest.close).toHaveBeenCalledWith(1008, "invalid Client message"); + }); }); diff --git a/tests/protocol.test.ts b/tests/protocol.test.ts index 3cbf4f307..516301079 100644 --- a/tests/protocol.test.ts +++ b/tests/protocol.test.ts @@ -51,17 +51,6 @@ const qualitySettings = { } as const; describe("runtime capabilities", () => { - it("keeps progress opt-in optional and strict for either signaling role", () => { - for (const role of ["host", "viewer"] as const) { - const identity = { type: "authenticate", protocol: SIGNALING_PROTOCOL, roomId, - clientId: "client_12345678", role, ...(role === "host" ? { token } : {}) }; - expect(clientMessageSchema.safeParse(identity).success).toBe(true); - for (const connectionAttemptProgress4 of [true, false, null, 4, "true"]) { - expect(clientMessageSchema.safeParse({ ...identity, connectionAttemptProgress4 }).success) - .toBe(connectionAttemptProgress4 === true); - } - } - }); it("defaults missing capabilities off and ignores unknown descriptors", () => { expect(runtimeCapabilitiesSchema.parse({})).toEqual({ sfu: false, natPrediction: false, roomInteractions: false, hostRoomSession: false }); expect(runtimeCapabilitiesSchema.parse({ sfu: true, extra: { enabled: true } })) @@ -70,8 +59,6 @@ describe("runtime capabilities", () => { .toEqual({ sfu: false, natPrediction: true, roomInteractions: false, hostRoomSession: false }); expect(runtimeCapabilitiesSchema.parse({ sfu: true, natPrediction: true })) .toEqual({ sfu: true, natPrediction: true, roomInteractions: false, hostRoomSession: false }); - expect(runtimeCapabilitiesSchema.parse({ connectionAttemptProgress4: true })) - .toEqual({ sfu: false, natPrediction: false, connectionAttemptProgress4: true, roomInteractions: false, hostRoomSession: false }); expect(runtimeCapabilitiesSchema.parse({ sfu: true, sfuOnly: true }).sfuOnly).toBe(true); }); @@ -79,7 +66,7 @@ describe("runtime capabilities", () => { it("rejects malformed known capabilities and non-object responses", () => { for (const value of [null, [], true, { sfu: null }, { sfu: "true" }, { natPrediction: null }, { natPrediction: 1 }, { sfuOnly: null }, { sfuOnly: "true" }, - { connectionAttemptProgress4: null }, { connectionAttemptProgress4: 4 }]) { + ]) { expect(runtimeCapabilitiesSchema.safeParse(value).success).toBe(false); } }); @@ -138,6 +125,23 @@ const qualityEvidence = { } as const; describe("client signaling protocol", () => { + it("keeps full observations with wide numeric values within the message budget", () => { + for (const floating of [Number.MAX_SAFE_INTEGER, 1.2345678901234567e-100]) { + const metrics = Object.fromEntries(Object.entries(qualityEvidence.metrics).map(([key, value]) => [key, + typeof value !== "number" ? value : key.endsWith("Percent") ? 99.99999999999999 : + ["width", "height", "packetsReceivedDelta", "packetsLostDelta", "framesDecodedDelta", + "framesDroppedDelta", "freezeCountDelta", "pauseCountDelta", "audioConcealmentEventsDelta"].includes(key) + ? Number.MAX_SAFE_INTEGER : floating, + ])); + const message = { ...qualityEvidence, sequence: Number.MAX_SAFE_INTEGER, + guard: { connectionId: "a".repeat(128), routeRevision: MAX_MEDIA_ROUTE_REVISION, + presentationEpoch: Number.MAX_SAFE_INTEGER }, + metrics: { ...metrics, audioCodec: `audio/${"a".repeat(32)}` } }; + expect(clientMessageSchema.safeParse(message).success).toBe(true); + expect(Buffer.byteLength(JSON.stringify(message))).toBeLessThanOrEqual(MAX_VIEWER_QUALITY_EVIDENCE_BYTES); + } + }); + it("uses the current strict signaling generation", () => { expect(SIGNALING_PROTOCOL).toBe("piik-v24"); }); @@ -752,7 +756,7 @@ describe("client signaling protocol", () => { ).toBe(false); }); - it("accepts only strict, bounded quality settings", () => { + it("separates numeric requests from capture implementation limits", () => { expect(DEFAULT_QUALITY_SETTINGS).not.toHaveProperty("videoCodec"); expect( clientMessageSchema.safeParse({ @@ -782,13 +786,23 @@ describe("client signaling protocol", () => { }).success, ).toBe(true); } - for (const invalid of [ - { ...qualitySettings, resolution: "2160p" }, + for (const next of [ + { ...qualitySettings, resolution: { width: 16384, height: 8192 } }, + { ...qualitySettings, resolution: { width: 1921, height: 1081 } }, { ...qualitySettings, maxFramerate: 14 }, { ...qualitySettings, maxFramerate: 61 }, { ...qualitySettings, maxFramerate: 30.5 }, { ...qualitySettings, maxBitrate: 1_999_999 }, { ...qualitySettings, maxBitrate: 12_000_001 }, + ]) expect(clientMessageSchema.safeParse({ type: "set-quality-settings", qualitySettings: next }).success).toBe(true); + for (const invalid of [ + { ...qualitySettings, resolution: "2160p" }, + { ...qualitySettings, resolution: { width: 0x1_0000_0000, height: 1080 } }, + { ...qualitySettings, maxFramerate: 0 }, + { ...qualitySettings, maxFramerate: Infinity }, + { ...qualitySettings, maxFramerate: Number.MAX_SAFE_INTEGER + 1 }, + { ...qualitySettings, maxBitrate: 0 }, + { ...qualitySettings, maxBitrate: 0x1_0000_0000 }, { ...qualitySettings, maxBitrate: 5_000_000.5 }, { ...qualitySettings, degradationPreference: "automatic" }, legacyCodecQualitySettings, @@ -1543,7 +1557,7 @@ describe("server signaling protocol", () => { expect( serverMessageSchema.safeParse({ ...peerAssisted, - qualitySettings: { ...qualitySettings, maxBitrate: 20_000_000 }, + qualitySettings: { ...qualitySettings, maxBitrate: 0x1_0000_0000 }, }).success, ).toBe(false); const missingMode = { @@ -1575,7 +1589,7 @@ describe("server signaling protocol", () => { childPeerId: "child_12345678", connectionId: "connection_12345678", transport: "direct", qualityProbe: false, }; - for (const total of [3, 4]) { + for (const total of [1, 3, 4, 5, 20]) { for (let current = 1; current <= total; current++) { expect(preparedRouteCandidateSchema.safeParse({ ...candidate, connectionAttempt: { current, total }, @@ -1584,7 +1598,7 @@ describe("server signaling protocol", () => { } for (const connectionAttempt of [ { current: 0, total: 3 }, { current: 4, total: 3 }, - { current: 5, total: 4 }, { current: 1, total: 5 }, { current: 1.5, total: 3 }, + { current: 5, total: 4 }, { current: 1, total: 0 }, { current: 1, total: Number.MAX_SAFE_INTEGER + 1 }, { current: 1.5, total: 3 }, ]) { expect(preparedRouteCandidateSchema.safeParse({ ...candidate, connectionAttempt, diff --git a/tests/quality-preference.test.ts b/tests/quality-preference.test.ts index bc3c51e3e..1181b3751 100644 --- a/tests/quality-preference.test.ts +++ b/tests/quality-preference.test.ts @@ -33,4 +33,12 @@ describe("sharing quality preference", () => { expect(readPreferredQuality()).toEqual(DEFAULT_QUALITY_SETTINGS); expect(() => savePreferredQuality(DEFAULT_QUALITY_SETTINGS)).not.toThrow(); }); + + it("uses supported defaults without overwriting a preference from a stronger implementation", () => { + const stored = JSON.stringify({ ...DEFAULT_QUALITY_SETTINGS, maxFramerate: 120, maxBitrate: 40_000_000 }); + const setItem = vi.fn(); + vi.stubGlobal("window", { localStorage: { getItem: () => stored, setItem } }); + expect(readPreferredQuality()).toEqual(DEFAULT_QUALITY_SETTINGS); + expect(setItem).not.toHaveBeenCalled(); + }); }); diff --git a/tests/quality.test.ts b/tests/quality.test.ts index b08fb7dd5..29dd21bf6 100644 --- a/tests/quality.test.ts +++ b/tests/quality.test.ts @@ -7,6 +7,7 @@ import { captureDisplay, captureFramerateCeiling, videoCaptureLimits, + supportsCaptureProfile, configureScreenAudioSender, configureVideoSender, matchingQualityProfileId, @@ -68,7 +69,26 @@ describe("realtime quality controls", () => { expect(captureFramerateCeiling(QUALITY_RESOLUTIONS["1080p"], limited)).toBe(30); expect(captureFramerateCeiling(QUALITY_RESOLUTIONS["720p"], limited)).toBe(60); expect(videoCaptureLimits({ maxVideoResolution: { width: 8192, height: 8192 }, - maxVideoPixels: 8192 * 8192, maxVideoMacroblocksPerSecond: 9_000_000 })).toEqual(videoCaptureLimits()); + maxVideoPixels: 8192 * 8192, maxVideoMacroblocksPerSecond: 9_000_000, + maxVideoFramerate: 120, maxVideoBitrate: 40_000_000 })).toEqual(videoCaptureLimits()); + const slower = videoCaptureLimits({ maxVideoFramerate: 30, maxVideoBitrate: 5_000_000 }); + expect(supportsCaptureProfile(QUALITY_PROFILES["1080p30"], slower)).toBe(true); + expect(supportsCaptureProfile(QUALITY_PROFILES["1080p60"], slower)).toBe(false); + }); + + it("rejects unsupported requests before changing capture or asking for permission", async () => { + const { stream, videoTrack } = createVideoStream(); + const getDisplayMedia = vi.fn(async () => stream); + vi.stubGlobal("navigator", { mediaDevices: { getDisplayMedia } }); + for (const change of [{ maxFramerate: 120 }, { maxFramerate: 30.5 }, { maxBitrate: 40_000_000 }, + { resolution: { width: 32768, height: 32768 } }, { maxAudioBitrate: 510_000 }]) { + const requested = { ...QUALITY_PROFILES["1080p30"], ...change }; + expect(supportsCaptureProfile(requested)).toBe(false); + await expect(captureDisplay(requested)).rejects.toThrow("not supported"); + await expect(applyCaptureProfile(stream, requested)).rejects.toThrow("not supported"); + } + expect(getDisplayMedia).not.toHaveBeenCalled(); + expect(videoTrack.applyConstraints).not.toHaveBeenCalled(); }); it("keeps exactly three recommended profiles with 1080p30 as the default", () => { diff --git a/tests/sender-quality-evidence.test.ts b/tests/sender-quality-evidence.test.ts index 541f95fa8..73b86cc18 100644 --- a/tests/sender-quality-evidence.test.ts +++ b/tests/sender-quality-evidence.test.ts @@ -38,6 +38,35 @@ function report( } describe("native sender quality evidence", () => { + it.each(["direct", "publication"])("preserves %s observations independently of control evidence", (path) => { + invalidateSenderQualityEvidence(); + const metrics = { + ...EMPTY_METRICS, sampleTimestampMs: 1_000, sampleWindowMs: 2_000, + rtpStatsId: "outbound-1", trackIdentifier: "track-1", intervalFramesEncoded: 960, + nativeEdgeQualityState: "healthy" as const, qualityLimitationReason: "none", + availableOutgoingKbps: 1_000_000, bitrateKbps: 120_000, framesPerSecond: 480, + rttMs: NaN, packetLossPercent: 101, + }; + const emit = () => path === "direct" + ? senderQualityEvidenceFromSnapshot({ + peerId: "viewer_12345678", connectionId: "connection_12345678", + connectionState: "connected", iceConnectionState: "connected", error: null, metrics, + }, 2) + : sfuPublisherQualityEvidenceFromMetrics(metrics, 2, "publication_12345678"); + expect(emit()?.state).toBe("unknown"); + metrics.sampleTimestampMs += 2_000; + expect(emit()).toMatchObject({ state: "healthy", diagnostics: { + availableOutgoingKbps: 1_000_000, bitrateKbps: 120_000, framesPerSecond: 480, + rttMs: null, packetLossPercent: null, + } }); + metrics.sampleTimestampMs += 2_000; + metrics.intervalFramesEncoded = NaN; + expect(emit()?.state).toBe("unknown"); + metrics.intervalFramesEncoded = 960; + metrics.sampleTimestampMs += 2_000; + expect(emit()?.state).toBe("healthy"); + }); + it("uses actual Native publication sent frames without inventing encoder work", () => { invalidateSenderQualityEvidence(); const metrics = { diff --git a/tests/viewer-quality-evidence.test.ts b/tests/viewer-quality-evidence.test.ts index b0706d75e..a3f4cbe5e 100644 --- a/tests/viewer-quality-evidence.test.ts +++ b/tests/viewer-quality-evidence.test.ts @@ -270,12 +270,12 @@ describe("viewer quality evidence", () => { natTraversalPath: "predicted", width: null, height: null, - bitrateKbps: null, + bitrateKbps: 200_000, packetsLostDelta: null, - rttMs: null, - audioBitrateKbps: null, + rttMs: 70_000, + audioBitrateKbps: 20_000, audioPacketLossPercent: null, - audioVideoPlayoutDeltaMs: null, + audioVideoPlayoutDeltaMs: -70_000, audioCodec: null, codecParameters: null, }); @@ -287,6 +287,15 @@ describe("viewer quality evidence", () => { expect(JSON.stringify(bounded)).not.toContain("useinbandfec=1"); expect(JSON.stringify(bounded)).not.toContain("192.0.2.10"); expect(JSON.stringify(bounded)).not.toContain("203.0.113.20"); + expect(qualityEvidenceWindowFromMetrics(receiveMetrics({ + framesPerSecond: Infinity, + bitrateKbps: Number.MAX_SAFE_INTEGER + 1, + audioVideoPlayoutDeltaMs: NaN, + intervalFramesDecoded: 1.5, + }))?.metrics).toMatchObject({ + framesPerSecond: null, bitrateKbps: null, + audioVideoPlayoutDeltaMs: null, framesDecodedDelta: null, + }); }); it("baselines each presentation epoch and keeps rate limits across it", () => { From be80b3a5c6f4bd0744a2c916a23d4a5b1321112d Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:33:00 +0800 Subject: [PATCH 152/262] feat(media): admit 4K60 with automatic H264 levels --- docs/research/browser-screen-audio-quality.md | 7 +++ docs/research/native-client-media.md | 48 +++++++++++++------ docs/standards/media-quality.md | 18 +++++-- docs/status.md | 2 +- docs/todo.md | 20 +++++--- internal/app/mediaedge/engine.go | 2 +- internal/app/mediaedge/receiver.go | 14 +----- internal/app/mediaedge/receiver_test.go | 5 +- internal/app/mediaedge/relay_input.go | 2 +- internal/app/mediaedge/relay_input_test.go | 4 ++ internal/app/nativecapture/protocol.go | 21 ++++++-- internal/app/nativecapture/protocol_test.go | 6 ++- internal/app/nativecontrol/session_test.go | 6 +-- internal/app/nativehost/session.go | 15 +----- internal/app/nativehost/session_test.go | 4 +- native/capture/darwin/main.swift | 7 +-- native/capture/linux/main.c | 4 +- native/capture/windows/adaptive_encoder.cpp | 2 +- native/capture/windows/h264_encoder.cpp | 21 ++++---- native/capture/windows/h264_encoder.h | 20 +------- native/capture/windows/main.cpp | 6 +-- native/capture/windows/output_worker.test.cpp | 26 ++++++---- native/capture/windows/video_bounds.h | 4 +- scripts/app-native-host-gate.ts | 2 +- scripts/encoded-group-decode.mjs | 2 +- src/client/media/quality.ts | 4 +- src/client/native/native-viewer-peer.ts | 2 +- tests/native-viewer-peer.test.ts | 3 +- tests/quality.test.ts | 29 +++++++++-- 29 files changed, 185 insertions(+), 121 deletions(-) diff --git a/docs/research/browser-screen-audio-quality.md b/docs/research/browser-screen-audio-quality.md index 23cefb767..59cefb5e4 100644 --- a/docs/research/browser-screen-audio-quality.md +++ b/docs/research/browser-screen-audio-quality.md @@ -62,6 +62,13 @@ preset contract. Candidate numeric requests and supported limits are owned by [media quality](../standards/media-quality.md#screen-audio); these measurements do not establish audible improvement or accept a public release. +For scope review on 2026-10-11, [RFC 7587](https://www.rfc-editor.org/rfc/rfc7587.html#section-3.1.1) +recommends 64..128 kbps for fullband stereo music with 20 ms frames, and +[LiveKit's high-quality stereo preset](https://github.com/livekit/client-sdk-js/blob/main/src/room/track/options.ts) +uses 128 kbps. These support retaining the current default; they are neither +codec maxima nor proof of quality for every source. Higher custom settings +remain subject to the measured packet and receiver bounds below. + The existing Go Opus encoder accepted live 128/192/256/320/96/128 kbps changes without replacement. A 48 kHz stereo tone/noise input continued decoding into 960 samples per channel; the existing deterministic tests also verify stereo diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 04152e8d2..c68e01a7d 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -592,27 +592,48 @@ App discovery advertises size, pixel-count, macroblock-rate, frame-rate and bitr size/rate combinations as operation failures without retiring the control session. Removing an enum is not permission to allocate arbitrary media resources. -The Windows H.264 UI gate switches from 1080p60 to the 4K ceiling with the -30 fps limit, then applies a non-preset 960x600 ceiling during the live share. -It observes fitted output without replacing the Viewer media object. The -captured window is smaller than 4K, so this verifies settings/lifecycle, not -physical 4K capture. +The Windows H.264 UI gate switches from 1080p60 to the 4K ceiling, then applies +a non-preset 960x600 ceiling during the live share. Its earlier 30 fps run +observed fitted output without replacing the Viewer media object. The gate now +expects the 60 fps control and needs rerunning. The captured window is smaller +than 4K, so this verifies settings/lifecycle, not physical 4K capture. Preset changes, paused updates, source recovery/replacement and subsequent sharing still pass. Value-based comparison prevents an identical deserialized size from turning an audio-only edit into a video update. Numeric preference round-trips and malformed/unsupported requests have separate boundary checks. -Native bounds now admit 3840x2160 and 2160x3840 at 30 fps. The per-axis limit and +Native bounds admit 3840x2160 and 2160x3840 at 60 fps. The per-axis limit and pixel total remain independent; an arbitrary 3840-square picture is rejected. -The output throughput ceiling uses H.264 Level 5.1's macroblock rate, including +The output throughput ceiling uses H.264 Level 5.2's macroblock rate, including alignment for non-preset sizes. Coded padding and decoded reference buffers have their own pre-allocation bounds, following [H.264 Table A-1](https://www.itu.int/rec/T-REC-H.264) (also implemented in [FFmpeg's level table](https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/h264_levels.c)). -The Windows decoder's advertised support remains -[Level 5.1](https://learn.microsoft.com/en-us/windows/win32/medfound/h-264-video-decoder). -An isolated Level 5.2 trial was rejected by this machine's hardware encoder at -output-type setup; it did not reach the decoder. The candidate therefore keeps -Level 5.1 and rejects 4K60 before allocating an encoder. +Level 5.2 raises throughput without raising Level 5.1's coded-frame or +reference-picture storage bounds. + +A 2026-10-11 control corrected the earlier 4K30 conclusion: explicitly forcing +Level 5.2 failed this hardware encoder's output-type setup, but leaving the +level unset encoded the same 4K60 inputs as Constrained Baseline Level 5.2. +[Microsoft recommends automatic level selection](https://learn.microsoft.com/en-us/windows/win32/medfound/h-264-video-encoder). +The production configuration follows that advice and validates actual SPS tools +and level through 5.2, retaining stable SPS identity and recovery-unit checks. +Local Media Foundation hardware/software decode delivered all 60 frames in both +4K orientations, with a passing 1080p60 control. Chrome also decoded all 60 +production-encoder frames per orientation; production H.264 and VP8 workers +passed a 4K-to-1080p-to-4K source sequence at the 60 fps setting. The normal +Go/native encoded-input path also derived lower outputs from these inputs. +This disproves a blanket 4K30 +restriction; the combined encode/decode fixture does not establish sustained +60 fps on every device. Microsoft's older decoder documentation lists +[Level 5.1](https://learn.microsoft.com/en-us/windows/win32/medfound/h-264-video-decoder), +so local success is not a cross-device guarantee. + +A separate Chrome 152 canvas/PeerConnection control decoded 3840x2160 near +60 fps with both H.264 and VP8. Its H.264 SDP advertised Level 3.1, consistent +with [WebRTC's built-in capability list](https://webrtc.googlesource.com/src/+/refs/heads/main/modules/video_coding/codecs/h264/h264.cc). +That static label alone does not establish a device's observed limit. Piik does +not add an independent quality ladder or infer universal decoder support from +codec matching; limited receivers and real routes still need acceptance. Decoder format checks must distinguish storage from visible content. On 2026-10-11, the Windows decoder rejected a valid 3840x2160 picture stored in a @@ -829,8 +850,7 @@ including narrow screens, both themes and visual mode. Start with numeric audio through the existing pipeline, then portrait/ultrawide geometry and 4K as a second concrete use case. A 4K path must review all platform -capture/decoder bounds, encoded-frame limits and SFU admission; Windows H.264's -current level ceiling does not cover 4K60. Admission and output must be tested +capture/decoder bounds, encoded-frame limits and SFU admission. Admission and output must be tested at the actual profile, not inferred from a codec list. Check live changes, late/limited Viewers and relays, failure cleanup and resource rejection before freezing the 2.0 contract. Prefer a clear 1.x mismatch over a general legacy diff --git a/docs/standards/media-quality.md b/docs/standards/media-quality.md index 75ddae3c6..e04f7d490 100644 --- a/docs/standards/media-quality.md +++ b/docs/standards/media-quality.md @@ -139,7 +139,9 @@ Larger wire-valid requests do not authorize an adapter to exceed its local capture, codec or allocation limits. Numeric requests use representational bounds rather than the current control ranges. Current capture still admits integer 15..60 fps and 2..12 Mbps, further -limited by picture size. App descriptions may advertise stronger capabilities; +limited by picture size. These are current product controls, not universal codec +limits or a minimum for framework-driven output. Codec and resource budgets +apply separately. App descriptions may advertise stronger capabilities; controls use their intersection with local support. An unsupported request is an operation failure, not malformed signaling. An unsupported saved preference does not seed capture or overwrite its stored value with a fallback. @@ -149,6 +151,8 @@ maintain-framerate` retain their controls. Display video uses the standard Native H264 retains Constrained Baseline. Capture validates the coding-tool subset under RFC 6184, including equivalent Constrained Baseline constraint bytes, and reports the actual SPS profile/level; one vendor's byte spelling is not the contract. +The platform encoder selects its level from the requested format. Validate the +actual supported level instead of forcing a calculated minimum into the encoder. Native ingress and relay accept only a negotiated profile/packetization compatible with their unchanged encoded downstream source. A codec name alone is not enough; same-source renegotiation must preserve that boundary before altering the live peer. @@ -159,8 +163,8 @@ large input. The candidate Native adapters admit even visible dimensions up to 3840 per axis and 3840x2160 total pixels. Their shared output-rate bound follows H.264 -Level 5.1: 983,040 16x16 macroblocks per second, with at most 60 fps. This admits -landscape/portrait 4K30 and existing 1440p60, not 4K60. Decoder alignment has a +Level 5.2: 2,073,600 16x16 macroblocks per second, with at most 60 fps. This admits +landscape/portrait 4K60 and existing 1440p60. Decoder alignment has a separate coded-storage bound; reference-picture storage is bounded before Native derivation. These are admission limits, not a device performance promise. App discovery reports size, pixel-count, frame-rate and bitrate limits through the @@ -378,6 +382,14 @@ application-defined whole-room lowest-common-denominator target. ## Screen Audio +The candidate supports mono and stereo Opus content through the existing audio +track. Mono inputs remain valid; Browser and Native mixing use a stereo output. +A stereo receive preference or two-channel declaration does not prove distinct +source channels. [Fidelity evidence](../research/media-fidelity.md#stereo) owns +decoded-channel checks. Surround delivery is outside the candidate; it needs +capture, channel-layout, mixing, codec and playback support, not only a channel +count setting. + Display audio uses `contentHint = "music"`. The 2.0 candidate keeps 64/128/192 kbps shortcuts and the 128 kbps default, with a custom numeric ceiling in the same sound settings. Current Browser/native senders support 6..320 kbps; the numeric diff --git a/docs/status.md b/docs/status.md index fe29483df..a277813b8 100644 --- a/docs/status.md +++ b/docs/status.md @@ -67,7 +67,7 @@ capture v7 and SQLite schema 2 remain unchanged. Matched candidate App/Web/Server builds are required; 1.x participants are rejected by the existing compatibility checks. Custom width/height controls apply combined size and frame-rate limits, admitting -4K30. Windows and Linux output preserve source shape within the selected ceiling. +4K60. Windows and Linux output preserve source shape within the selected ceiling. The [media assessment](./research/native-client-media.md#media-capability-extension-assessment) owns local codec, derived-output, SFU, Browser/Native fanout and live-control evidence. The macOS geometry implementation still needs SDK compilation and diff --git a/docs/todo.md b/docs/todo.md index bd4c8efa3..385dc8cd4 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -26,8 +26,9 @@ Git/PRs own completed history. - [ ] **Reconcile open PRs with the final candidate.** [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s 256/320 kbps request and [#469](https://github.com/TNTcraftHIM/Piik/pull/469)'s 4K request overlap the - numeric-media candidate; neither PR has been merged. The latter's 4K60 and - 40 Mbps expansion needs separate evidence and is not accepted scope. Preserve + numeric-media candidate; neither PR has been merged. 4K60 is included in + candidate admission and still needs route/device acceptance; 40 Mbps needs + separate evidence and is not accepted scope. Preserve contributor credit and explain the disposition at integration. Rewrite [#459](https://github.com/TNTcraftHIM/Piik/pull/459)'s earlier maintenance scope and compatibility claims around the final 2.0 result before integration. @@ -74,21 +75,26 @@ Implementation needs a concrete useful result within the product model. actual output. Numeric request, capability and observation boundaries are now separate from implementation admission. Keep receiver, codec/alignment and resource budgets intact; opening an output range still needs end-to-end evidence. - Assess higher audio output against packet-duration/size and receiver support; - removing the current 320 kbps sender guard alone is not sufficient. Numeric - audio and video controls are implemented; the assessment + Retain the current audio sender ceiling and bounded video controls; extending + their operating range needs a concrete use case and end-to-end evidence. + Numeric audio and video controls are implemented; the assessment owns local capture, codec, derived-output, SFU and bounded mixed-route evidence. Remaining receiver acceptance needs physical limited decoders and camera orientation behavior, beyond the bounded SDP-limit and mixed-route controls. Browser display capture, Browser relay and Browser-to-Native fanout now have bounded high-resolution evidence. macOS geometry SDK/hardware checks and - physical 4K/game capture remain open. Preserve safety limits and + physical 4K/game capture remain open. Repeat live Native-to-Browser, derived + relay and SFU checks at 4K60; local codec and Browser loopback success does not + establish sustained full-route performance. Preserve safety limits and framework-owned adaptation. The [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment) retain explicit receive-limit enforcement and the current 320 kbps sender ceiling. Physical listening, mobile and field-network acceptance remain open; research owns the bounded real-Opus impairment and video-recovery control. - Additional codecs are outside this candidate. Later inclusion needs complete + Validate the existing mono/stereo and Windows HDR-to-SDR paths under the + [fidelity boundary](./research/media-fidelity.md); surround delivery and + end-to-end HDR are outside this candidate. Additional codecs are also outside + this candidate. Later inclusion needs complete encode/decode/forward support and a late-Viewer policy; selecting a codec cannot transcode an existing publication. Final package acceptance must retain the verified live-change, relay/SFU and limited-receiver diff --git a/internal/app/mediaedge/engine.go b/internal/app/mediaedge/engine.go index 351494c96..1e93d988f 100644 --- a/internal/app/mediaedge/engine.go +++ b/internal/app/mediaedge/engine.go @@ -18,7 +18,7 @@ import ( // Use the browser-standard spelling of Constrained Baseline in SDP. Pion's // codec ordering prefers literal profile matches over equivalent constraint bits. -const H264ProfileLevelID = "42e033" +const H264ProfileLevelID = "42e034" var h264Capability = webrtc.RTPCodecCapability{ MimeType: webrtc.MimeTypeH264, diff --git a/internal/app/mediaedge/receiver.go b/internal/app/mediaedge/receiver.go index e9d01a30b..300972876 100644 --- a/internal/app/mediaedge/receiver.go +++ b/internal/app/mediaedge/receiver.go @@ -10,7 +10,7 @@ import ( "sync" "github.com/TNTcraftHIM/Piik/internal/app/nativeaudio" - "github.com/TNTcraftHIM/Piik/internal/media/encoded" + "github.com/TNTcraftHIM/Piik/internal/app/nativecapture" "github.com/pion/rtcp" "github.com/pion/sdp/v3" "github.com/pion/webrtc/v4" @@ -465,15 +465,5 @@ func forwardableH264(fmtp string) bool { packetization = strings.TrimSpace(value) } } - level, valid := encoded.H264ConstrainedBaselineLevel(profile) - if !valid || packetization != "1" { - return false - } - // Receiving may include smaller Browser encodes than native capture presets. - switch level { - case 10, 11, 12, 13, 20, 21, 22, 30, 31, 32, 40, 41, 42, 50, 51: - return true - default: - return false - } + return packetization == "1" && nativecapture.SupportedH264ProfileLevelID(profile) } diff --git a/internal/app/mediaedge/receiver_test.go b/internal/app/mediaedge/receiver_test.go index 46e3c0a5d..e5bc82613 100644 --- a/internal/app/mediaedge/receiver_test.go +++ b/internal/app/mediaedge/receiver_test.go @@ -180,8 +180,9 @@ func TestReceiverNegotiatesForwardableH264(t *testing.T) { {"missing-profile", []string{""}, ""}, {"equivalent-baseline", []string{"42c01f"}, "42c01f"}, {"small-browser-output", []string{"42e00b"}, "42e00b"}, - {"level-ceiling", []string{"42e033"}, "42e033"}, - {"over-ceiling", []string{"42e034"}, ""}, + {"level-5.1", []string{"42e033"}, "42e033"}, + {"level-ceiling", []string{"42e034"}, "42e034"}, + {"over-ceiling", []string{"42e03c"}, ""}, {"invalid-level", []string{"42e035"}, ""}, {"invalid-constraints", []string{"42e11f"}, ""}, {"mode-zero", []string{"42e01f;packetization-mode=0"}, ""}, diff --git a/internal/app/mediaedge/relay_input.go b/internal/app/mediaedge/relay_input.go index a1c98dfe5..b9959c723 100644 --- a/internal/app/mediaedge/relay_input.go +++ b/internal/app/mediaedge/relay_input.go @@ -159,7 +159,7 @@ func relayH264Size(nal []byte) (uint32, uint32, error) { bufferedFrames = max(bufferedFrames, sps.VUI.BitstreamRestriction.MaxDecFrameBuffering, sps.VUI.BitstreamRestriction.MaxNumReorderFrames) } - // Level 5.1 also bounds the decoded-picture buffer. A per-frame size check + // Level 5.2 also bounds the decoded-picture buffer. A per-frame size check // alone would let a larger picture multiply the old 16-reference allocation. if bufferedFrames > 16 || width*height/256*uint64(bufferedFrames) > 184_320 { return 0, 0, errors.New("native H264 input reference storage exceeds its bound") diff --git a/internal/app/mediaedge/relay_input_test.go b/internal/app/mediaedge/relay_input_test.go index f44f397ea..b77c65a2c 100644 --- a/internal/app/mediaedge/relay_input_test.go +++ b/internal/app/mediaedge/relay_input_test.go @@ -108,9 +108,13 @@ func TestRelayH264AllocationBounds(t *testing.T) { {"portrait-within-bound", "6742c033da01100b5e5d", 1080, 1440}, {"4k", "6742c033da00f0010f90", 3840, 2160}, {"portrait-4k", "6742c033da008701e190", 2160, 3840}, + {"level-5.2-4k", "6742c034da00f0010f90", 3840, 2160}, + {"level-5.2-portrait-4k", "6742c034da008701e190", 2160, 3840}, {"padded-4k", "6742c033da00f00111f894", 3840, 2160}, {"4k-reference-buffer-limit", "6742c033f300780087c8", 3840, 2160}, {"4k-reference-buffer-overflow", "6742c033f380780087c8", 0, 0}, + {"level-5.2-reference-buffer-overflow", "6742c034f380780087c8", 0, 0}, + {"level-5.2-cropped-storage-overflow", "6742c034da00f001e1f8034940", 0, 0}, {"large-cropped-storage", "6742c033da00f001e1f8034940", 0, 0}, {"crop-overflow", "6742c033da0a37e000000300100000030004", 0, 0}, {"coded-size-overflow", "6742c033da000003000040000003000d90", 0, 0}, diff --git a/internal/app/nativecapture/protocol.go b/internal/app/nativecapture/protocol.go index b3cd65741..1b44917d2 100644 --- a/internal/app/nativecapture/protocol.go +++ b/internal/app/nativecapture/protocol.go @@ -17,10 +17,10 @@ const ( maxControlBytes = 64 MaxVideoDimension = 3840 MaxVideoPixels = 3840 * 2160 - MaxVideoMacroblocksPerSecond = 983_040 + MaxVideoMacroblocksPerSecond = 2_073_600 MaxVideoFramerate = 60 // Coded pictures may include alignment outside the visible crop. Keep this - // separate from visible dimensions and within H.264 Level 5.1 storage. + // separate from visible dimensions and within H.264 Level 5.2 storage. MaxCodedVideoDimension = 4096 MaxCodedVideoPixels = 36_864 * 16 * 16 ) @@ -33,7 +33,7 @@ func ValidVideoSize(width, height uint32) bool { } // ValidVideoFormat keeps both current encoders within the shared receive -// ceiling (H.264 Level 5.1), including non-preset shapes and frame rates. +// ceiling (H.264 Level 5.2), including non-preset shapes and frame rates. func ValidVideoFormat(width, height, framerate uint32) bool { return ValidVideoSize(width, height) && framerate >= 1 && framerate <= MaxVideoFramerate && uint64((width+15)/16)*uint64((height+15)/16)*uint64(framerate) <= MaxVideoMacroblocksPerSecond @@ -45,6 +45,21 @@ func ValidCodedVideoSize(width, height uint64) bool { width*height <= MaxCodedVideoPixels } +// SupportedH264ProfileLevelID bounds capture and receive to the same coding +// tools. Automatic encoders may select a lower level for a smaller output. +func SupportedH264ProfileLevelID(value string) bool { + level, valid := encoded.H264ConstrainedBaselineLevel(value) + if !valid { + return false + } + switch level { + case 10, 11, 12, 13, 20, 21, 22, 30, 31, 32, 40, 41, 42, 50, 51, 52: + return true + default: + return false + } +} + type FrameKind uint8 const ( diff --git a/internal/app/nativecapture/protocol_test.go b/internal/app/nativecapture/protocol_test.go index 8ef21f0be..1e7806a3d 100644 --- a/internal/app/nativecapture/protocol_test.go +++ b/internal/app/nativecapture/protocol_test.go @@ -294,6 +294,8 @@ func TestVideoProfileUsesTheProductBounds(t *testing.T) { {Width: 720, Height: 1280, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, {Width: 3840, Height: 2160, Framerate: 30, Bitrate: 12_000_000, Preference: "balanced"}, {Width: 2160, Height: 3840, Framerate: 30, Bitrate: 12_000_000, Preference: "balanced"}, + {Width: 3840, Height: 2160, Framerate: 60, Bitrate: 12_000_000, Preference: "balanced"}, + {Width: 2160, Height: 3840, Framerate: 60, Bitrate: 12_000_000, Preference: "balanced"}, {Width: 3000, Height: 2764, Framerate: 30, Bitrate: 12_000_000, Preference: "balanced"}, {Width: 4, Height: 4, Framerate: 15, Bitrate: 2_000_000, Preference: "balanced"}, } { @@ -316,8 +318,8 @@ func TestVideoProfileUsesTheProductBounds(t *testing.T) { {Width: 2160, Height: 3842, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, {Width: 3840, Height: 2162, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, {Width: 3000, Height: 2766, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, - {Width: 3840, Height: 2160, Framerate: 31, Bitrate: 3_000_000, Preference: "balanced"}, - {Width: 2160, Height: 3840, Framerate: 60, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 3840, Height: 2160, Framerate: 61, Bitrate: 3_000_000, Preference: "balanced"}, + {Width: 2160, Height: 3840, Framerate: 120, Bitrate: 3_000_000, Preference: "balanced"}, {Width: ^uint32(0), Height: ^uint32(0), Framerate: ^uint32(0), Bitrate: 3_000_000, Preference: "balanced"}, {Width: 1920, Height: 1080, Framerate: 14, Bitrate: 5_000_000, Preference: "balanced"}, {Width: 1920, Height: 1080, Framerate: 30, Bitrate: 12_000_001, Preference: "balanced"}, diff --git a/internal/app/nativecontrol/session_test.go b/internal/app/nativecontrol/session_test.go index 51868f6db..8cdc4969c 100644 --- a/internal/app/nativecontrol/session_test.go +++ b/internal/app/nativecontrol/session_test.go @@ -670,14 +670,14 @@ func TestQualitySettingsMapOnceIntoNativeMedia(t *testing.T) { t.Fatalf("numeric size outside UI presets was not preserved: %+v", profile) } settings.Resolution = protocol.QualityResolution{Width: 3840, Height: 2160} - if nativeQualityProfile(settings).Valid() { - t.Fatal("unsupported 4K60 combination was accepted") + if !nativeQualityProfile(settings).Valid() { + t.Fatal("4K60 was rejected") } settings.MaxFramerate = 30 if !nativeQualityProfile(settings).Valid() { t.Fatal("4K30 was rejected") } - for _, fps := range []protocol.Num{30.5, 120, 0x1_0000_0000 + 30} { + for _, fps := range []protocol.Num{30.5, 61, 120, 0x1_0000_0000 + 30} { settings.MaxFramerate = fps if nativeQualityProfile(settings).Valid() { t.Fatalf("unsupported fps %v was narrowed into a valid profile", fps) diff --git a/internal/app/nativehost/session.go b/internal/app/nativehost/session.go index 4868a3b2c..126238e3d 100644 --- a/internal/app/nativehost/session.go +++ b/internal/app/nativehost/session.go @@ -1226,7 +1226,7 @@ func decodeCaptureState(payload []byte) (CaptureState, error) { } original := state.Outputs[min(1, len(state.Outputs)-1)] if state.State == "active" && - ((state.Codec == "h264" && !validH264ProfileLevelID(state.ProfileLevelID)) || + ((state.Codec == "h264" && !nativecapture.SupportedH264ProfileLevelID(state.ProfileLevelID)) || (state.Codec == "vp8" && state.ProfileLevelID != "") || state.Width != original.Width || state.Height != original.Height || @@ -1249,19 +1249,6 @@ func (state CaptureState) applyBackend(options *nativecapture.VideoOptions) { } } -func validH264ProfileLevelID(value string) bool { - level, valid := encoded.H264ConstrainedBaselineLevel(value) - if !valid { - return false - } - switch level { - case 30, 31, 32, 40, 41, 42, 50, 51: - return true - default: - return false - } -} - // Optional facts are emitted only to a page which opted into mixed audio. func (session *Session) SourceAudio() *bool { if session.mixer == nil { diff --git a/internal/app/nativehost/session_test.go b/internal/app/nativehost/session_test.go index 9ba885783..5c7b6b7ae 100644 --- a/internal/app/nativehost/session_test.go +++ b/internal/app/nativehost/session_test.go @@ -430,14 +430,14 @@ func TestCaptureStateRejectsUnknownOrUnattributedState(t *testing.T) { } func TestCaptureStateValidatesH264ToolsRatherThanConstraintSpelling(t *testing.T) { - for _, profile := range []string{"42c01f", "42e01f", "42E033", "4d801f", "58c02a"} { + for _, profile := range []string{"42c01f", "42e01f", "42E033", "4d801f", "58c02a", "42c00a", "42c016", "42c034"} { state, err := decodeCaptureState(captureStatePayload(t, `{"state":"active","hardwareOnly":true,"codec":"h264","profileLevelId":"`+profile+`","width":1280,"height":720,"fps":30}`)) if err != nil || state.ProfileLevelID != profile { t.Fatalf("equivalent Constrained Baseline %q was rejected or rewritten: %+v, %v", profile, state, err) } } - for _, profile := range []string{"42001f", "4d001f", "64001f", "42e11f", "42e01", "42e01fg", "42e0gg", "42e034", "42e035", "42e03c"} { + for _, profile := range []string{"42001f", "4d001f", "64001f", "42e11f", "42e01", "42e01fg", "42e0gg", "42e000", "42e035", "42e03c"} { if _, err := decodeCaptureState(captureStatePayload(t, `{"state":"active","hardwareOnly":true,"codec":"h264","profileLevelId":"`+profile+`","width":1280,"height":720,"fps":30}`)); err == nil { t.Fatalf("incompatible profile/level %q was admitted", profile) diff --git a/native/capture/darwin/main.swift b/native/capture/darwin/main.swift index 65f036c78..33d5975d9 100644 --- a/native/capture/darwin/main.swift +++ b/native/capture/darwin/main.swift @@ -20,7 +20,8 @@ private let maxVideoPixels = 3840 * 2160 private let maxCodedVideoDimension = 4096 private let maxCodedVideoPixels = 36_864 * 16 * 16 private let supportedH264Levels: Set = [ - 0x1e, 0x1f, 0x20, 0x28, 0x29, 0x2a, 0x32, 0x33, + 0x0a, 0x0b, 0x0c, 0x0d, 0x14, 0x15, 0x16, + 0x1e, 0x1f, 0x20, 0x28, 0x29, 0x2a, 0x32, 0x33, 0x34, ] private enum DegradationPreference: String { @@ -1497,9 +1498,9 @@ private func validVideoSize(_ width: Int, _ height: Int) -> Bool { } private func validVideoFormat(_ width: Int, _ height: Int, _ fps: Int32) -> Bool { - // Match the shared H.264 Level 5.1 receive ceiling, including numeric sizes. + // Match the shared H.264 Level 5.2 receive ceiling, including numeric sizes. validVideoSize(width, height) && fps >= 1 && fps <= 60 && - ((width + 15) / 16) * ((height + 15) / 16) * Int(fps) <= 983_040 + ((width + 15) / 16) * ((height + 15) / 16) * Int(fps) <= 2_073_600 } private func videoProfile(_ arguments: [String]) throws -> VideoProfile { diff --git a/native/capture/linux/main.c b/native/capture/linux/main.c index 575cc5443..f3e06da4b 100644 --- a/native/capture/linux/main.c +++ b/native/capture/linux/main.c @@ -379,9 +379,9 @@ static gboolean valid_video_size(guint width, guint height) { } static gboolean valid_video_format(guint width, guint height, guint fps) { - // Match the shared H.264 Level 5.1 receive ceiling, including numeric sizes. + // Match the shared H.264 Level 5.2 receive ceiling, including numeric sizes. return valid_video_size(width, height) && fps >= 1 && fps <= 60 && - (guint64)((width + 15) / 16) * ((height + 15) / 16) * fps <= 983040; + (guint64)((width + 15) / 16) * ((height + 15) / 16) * fps <= 2073600; } static gboolean parse_outputs(int count, char **values, guint start, CaptureRun *run) { diff --git a/native/capture/windows/adaptive_encoder.cpp b/native/capture/windows/adaptive_encoder.cpp index 8c5a4edc0..5b2942be5 100644 --- a/native/capture/windows/adaptive_encoder.cpp +++ b/native/capture/windows/adaptive_encoder.cpp @@ -508,7 +508,7 @@ class AdaptiveEncoder::Impl final : public webrtc::VideoSourceInterface GetSupportedFormats() const override { if (kind_ == OutputKind::vp8) return {webrtc::SdpVideoFormat("VP8")}; - return {webrtc::SdpVideoFormat("H264", {{"profile-level-id", ceiling_.profile_level_id()}, + return {webrtc::SdpVideoFormat("H264", {{"profile-level-id", kH264ProfileLevelId}, {"packetization-mode", "1"}, {"level-asymmetry-allowed", "1"}})}; } std::unique_ptr Create(const webrtc::Environment& env, diff --git a/native/capture/windows/h264_encoder.cpp b/native/capture/windows/h264_encoder.cpp index f1cab0512..8b00e96e2 100644 --- a/native/capture/windows/h264_encoder.cpp +++ b/native/capture/windows/h264_encoder.cpp @@ -17,12 +17,19 @@ namespace { constexpr DWORD kMaxEncodedSampleBytes = 4 * 1024 * 1024; } -bool VideoProfile::accepts_h264_profile_level_id(const std::string& value) const { +bool SupportsH264ProfileLevelId(const std::string& value) { if (value.size() != 6) return false; UINT32 parsed = 0; const auto end = value.data() + value.size(); const auto result = std::from_chars(value.data(), end, parsed, 16); - if (result.ec != std::errc{} || result.ptr != end || (parsed & 0xff) != h264_level()) return false; + if (result.ec != std::errc{} || result.ptr != end) return false; + switch (parsed & 0xff) { + case 10: case 11: case 12: case 13: case 20: case 21: case 22: + case 30: case 31: case 32: case 40: case 41: case 42: case 50: case 51: case 52: + break; + default: + return false; + } const auto idc = parsed >> 16; const auto constraints = (parsed >> 8) & 0xff; // RFC 6184 Table 5: these are the same Constrained Baseline tools, not @@ -359,8 +366,8 @@ ComPtr CreateOutputType( "output-type-interlace"); Check(type->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_Base), "output-type-profile"); - Check(type->SetUINT32(MF_MT_MPEG2_LEVEL, profile.h264_level()), - "output-type-level"); + // MF recommends automatic level selection. Forcing a valid level can still + // make hardware reject the type; validate the actual SPS instead. return type; } @@ -412,8 +419,6 @@ void ValidateMediaTypes( "output-type-bitrate"); RequireTypeU32(output.Get(), MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_Base, "output-type-profile"); - RequireTypeU32(output.Get(), MF_MT_MPEG2_LEVEL, profile.h264_level(), - "output-type-level"); UINT32 width = 0; UINT32 height = 0; @@ -703,8 +708,8 @@ EncodedAccessUnit LiveEncoder::Encode( Fail("bitstream-annexb", "live output is not Annex-B H264"); } if (nal.profile_level_id) { - if (!profile_.accepts_h264_profile_level_id(*nal.profile_level_id)) { - Fail("bitstream-profile", "hardware MFT changed the requested H.264 profile level"); + if (!SupportsH264ProfileLevelId(*nal.profile_level_id)) { + Fail("bitstream-profile", "hardware MFT emitted unsupported H.264 coding tools or level"); } if (profile_level_id_ && *profile_level_id_ != *nal.profile_level_id) { Fail("bitstream-profile-change", "hardware MFT changed SPS profile"); diff --git a/native/capture/windows/h264_encoder.h b/native/capture/windows/h264_encoder.h index dd7ed10f8..f1ce69358 100644 --- a/native/capture/windows/h264_encoder.h +++ b/native/capture/windows/h264_encoder.h @@ -56,27 +56,11 @@ struct VideoProfile final { if (preference == DegradationPreference::framerate) return 0; return 50; } - UINT32 h264_level() const { - const UINT64 macroblocks = - ((static_cast(width) + 15) / 16) * - ((static_cast(height) + 15) / 16); - const UINT64 macroblocks_per_second = macroblocks * frame_rate; - if (macroblocks <= 3'600 && macroblocks_per_second <= 108'000) return 31; - if (macroblocks <= 8'192 && macroblocks_per_second <= 245'760) return 40; - if (macroblocks <= 8'704 && macroblocks_per_second <= 522'240) return 42; - if (macroblocks <= 22'080 && macroblocks_per_second <= 589'824) return 50; - return 51; - } - std::string profile_level_id() const { - std::ostringstream output; - output << "42c0" << std::hex << std::setfill('0') << std::setw(2) - << h264_level(); - return output.str(); - } - bool accepts_h264_profile_level_id(const std::string& value) const; }; constexpr VideoProfile kDefaultVideoProfile{}; +inline constexpr char kH264ProfileLevelId[] = "42c034"; +bool SupportsH264ProfileLevelId(const std::string& value); struct Runtime final { Runtime() { diff --git a/native/capture/windows/main.cpp b/native/capture/windows/main.cpp index 9f1a5eae1..5183bd502 100644 --- a/native/capture/windows/main.cpp +++ b/native/capture/windows/main.cpp @@ -607,9 +607,9 @@ RunEvidence RunEncoder(const Adapter& adapter, const DeviceContext& device, Fail("codec-runtime-bitrate-effect", "live bitrate update did not lower and restore encoded output"); } - if (!kDefaultVideoProfile.accepts_h264_profile_level_id(*observed_profile)) { - Fail("bitstream-pinned-fmtp", - "SPS profile-level-id differs from the native media contract"); + if (!SupportsH264ProfileLevelId(*observed_profile)) { + Fail("bitstream-profile", + "SPS uses unsupported H.264 coding tools or level"); } return evidence; } diff --git a/native/capture/windows/output_worker.test.cpp b/native/capture/windows/output_worker.test.cpp index 2110b2ded..1c8f62a16 100644 --- a/native/capture/windows/output_worker.test.cpp +++ b/native/capture/windows/output_worker.test.cpp @@ -151,11 +151,19 @@ void CheckWorkerDimensions(ID3D11Device* device, SIZE content, SIZE expected) { void CheckH264ProfileCompatibility() { const VideoProfile profile; - for (const auto* value : {"42c01f", "42e01f", "42E01F", "4d801f", "58c01f"}) - assert(profile.accepts_h264_profile_level_id(value)); + for (const auto* value : {"42c01f", "42e01f", "42E01F", "4d801f", "58c01f", + "42c00a", "42c016", "42e01e", "42c02a", "42c033", "42c034"}) + assert(SupportsH264ProfileLevelId(value)); for (const auto* value : {"42001f", "4d001f", "64001f", "42e11f", "42e01", "42e01fg", - "42e0gg", "42e034", "42e01e", " 42c01"}) - assert(!profile.accepts_h264_profile_level_id(value)); + "42e0gg", "42e035", "42e03c", "42e000", " 42c01"}) + assert(!SupportsH264ProfileLevelId(value)); + Runtime runtime; + for (const auto size : {SIZE{1280, 720}, SIZE{3840, 2160}, SIZE{2160, 3840}}) { + const auto type = CreateOutputType(VideoProfile{ + static_cast(size.cx), static_cast(size.cy), 60, 12'000'000}); + UINT32 level = 0; + assert(type->GetUINT32(MF_MT_MPEG2_LEVEL, &level) == MF_E_ATTRIBUTENOTFOUND); + } HANDLE read = nullptr, write = nullptr; assert(CreatePipe(&read, &write, nullptr, 4096)); @@ -392,12 +400,9 @@ void CheckDecodedDisplayArea(ID3D11Device* device) { int main() { VideoProfile profile{2560, 1440, 60, 12'000'000}; ValidateVideoProfile(profile); - assert(profile.h264_level() == 51 && profile.profile_level_id() == "42c033"); - profile = VideoProfile{1920, 1080, 60, 8'000'000}; - assert(profile.h264_level() == 42); for (const SIZE size : {SIZE{1920, 1200}, SIZE{720, 1280}, SIZE{4, 4}, SIZE{3840, 2160}, SIZE{2160, 3840}, SIZE{3000, 2764}}) { - ValidateVideoProfile(VideoProfile{static_cast(size.cx), static_cast(size.cy), 30, 3'000'000}); + ValidateVideoProfile(VideoProfile{static_cast(size.cx), static_cast(size.cy), 60, 3'000'000}); } for (const SIZE size : {SIZE{0, 720}, SIZE{2, 720}, SIZE{1280, 2}, SIZE{1279, 720}, SIZE{1280, 719}, SIZE{3842, 2160}, SIZE{2160, 3842}, @@ -410,8 +415,9 @@ int main() { } } assert(piik::capture::ValidVideoFormat(3840, 2160, 30)); - assert(!piik::capture::ValidVideoFormat(3840, 2160, 31)); - assert(!piik::capture::ValidVideoFormat(2160, 3840, 60)); + assert(piik::capture::ValidVideoFormat(3840, 2160, 60)); + assert(piik::capture::ValidVideoFormat(2160, 3840, 60)); + assert(!piik::capture::ValidVideoFormat(3840, 2160, 61)); assert(!piik::capture::ValidVideoFormat(UINT32_MAX, UINT32_MAX, UINT32_MAX)); assert(piik::capture::ValidCodedVideoSize(3840, 2176)); assert(piik::capture::ValidCodedVideoSize(2176, 3840)); diff --git a/native/capture/windows/video_bounds.h b/native/capture/windows/video_bounds.h index 79fac850a..93648722e 100644 --- a/native/capture/windows/video_bounds.h +++ b/native/capture/windows/video_bounds.h @@ -6,7 +6,7 @@ namespace piik::capture { inline constexpr uint32_t kMaxVideoDimension = 3840; inline constexpr uint64_t kMaxVideoPixels = 3840 * 2160; -// H.264 Level 5.1 coded storage, including padding outside the visible crop. +// H.264 Level 5.2 coded storage, including padding outside the visible crop. inline constexpr uint32_t kMaxCodedVideoDimension = 4096; inline constexpr uint64_t kMaxCodedVideoPixels = 36'864 * 16 * 16; @@ -18,7 +18,7 @@ inline bool ValidVideoSize(uint32_t width, uint32_t height) { inline bool ValidVideoFormat(uint32_t width, uint32_t height, uint32_t fps) { return ValidVideoSize(width, height) && fps >= 1 && fps <= 60 && - static_cast((width + 15) / 16) * ((height + 15) / 16) * fps <= 983'040; + static_cast((width + 15) / 16) * ((height + 15) / 16) * fps <= 2'073'600; } inline bool ValidCodedVideoSize(uint32_t width, uint32_t height) { diff --git a/scripts/app-native-host-gate.ts b/scripts/app-native-host-gate.ts index c6c0f6d12..11abe0873 100644 --- a/scripts/app-native-host-gate.ts +++ b/scripts/app-native-host-gate.ts @@ -1037,7 +1037,7 @@ async function main(): Promise { const fps = document.querySelector('.lr-slider input[type="range"]'); const width = document.querySelector('.lr-video-size input[name="width"]'); const height = document.querySelector('.lr-video-size input[name="height"]'); - return fps?.max === '30' && fps.value === '30' && width?.value === '3840' && height?.value === '2160'; + return fps?.max === '60' && fps.value === '60' && width?.value === '3840' && height?.value === '2160'; })()`, deadline), Boolean, 10_000); stage = "native-4k-decode"; result.fourKQualityChanged = await waitForValue(deadline => evaluate(cdp!, viewer, `(() => { diff --git a/scripts/encoded-group-decode.mjs b/scripts/encoded-group-decode.mjs index 76cfcfa5b..68804635f 100644 --- a/scripts/encoded-group-decode.mjs +++ b/scripts/encoded-group-decode.mjs @@ -6,7 +6,7 @@ import { resolve } from "node:path"; const customInput = process.env.PIIK_ENCODED_OUTPUT; const codec = process.env.PIIK_ENCODED_CODEC ?? "vp8"; -assert.ok(["vp8", "avc1.42c033"].includes(codec), "Expected VP8 or constrained-baseline H264"); +assert.ok(["vp8", "avc1.42c033", "avc1.42c034"].includes(codec), "Expected VP8 or constrained-baseline H264"); const inputPath = customInput ? resolve(customInput) : new URL("../build/embedded-media/encoded-group.received.json", import.meta.url); const input = await readFile(inputPath); const streams = JSON.parse(input); diff --git a/src/client/media/quality.ts b/src/client/media/quality.ts index 779ac5722..2b87315e1 100644 --- a/src/client/media/quality.ts +++ b/src/client/media/quality.ts @@ -34,11 +34,11 @@ export interface VideoCaptureLimits { } // Current output boundary, independent of descriptive wire bounds and UI presets. -// The common H.264 Level 5.1 path admits 4K30 and 1440p60 in either orientation. +// H.264 Level 5.2 admits 4K60 in either orientation; actual output still adapts. export const VIDEO_CAPTURE_LIMITS: VideoCaptureLimits = { maxVideoResolution: { width: 3840, height: 3840 }, maxVideoPixels: 3840 * 2160, - maxVideoMacroblocksPerSecond: 983_040, + maxVideoMacroblocksPerSecond: 2_073_600, maxVideoFramerate: 60, maxVideoBitrate: 12_000_000, }; diff --git a/src/client/native/native-viewer-peer.ts b/src/client/native/native-viewer-peer.ts index aaea00220..2827efc79 100644 --- a/src/client/native/native-viewer-peer.ts +++ b/src/client/native/native-viewer-peer.ts @@ -831,5 +831,5 @@ function nativeH264Format(fmtp: string): boolean { return ((idc === 0x42 && (constraints & 0x4f) === 0x40) || (idc === 0x4d && (constraints & 0x8f) === 0x80) || (idc === 0x58 && (constraints & 0xcf) === 0xc0)) && - [10, 11, 12, 13, 20, 21, 22, 30, 31, 32, 40, 41, 42, 50, 51].includes(level); + [10, 11, 12, 13, 20, 21, 22, 30, 31, 32, 40, 41, 42, 50, 51, 52].includes(level); } diff --git a/tests/native-viewer-peer.test.ts b/tests/native-viewer-peer.test.ts index d0d42eac7..62fd3b554 100644 --- a/tests/native-viewer-peer.test.ts +++ b/tests/native-viewer-peer.test.ts @@ -272,7 +272,8 @@ it("admits one active sending H264 or VP8 video section, not unrelated SDP codec it.each([ ["42e01f", "1", true], ["42c01f", "1", true], ["4de01f", "1", true], ["58c01f", "1", true], - ["42e00b", "1", true], ["42e033", "1", true], ["42e034", "1", false], + ["42e00b", "1", true], ["42e033", "1", true], ["42e034", "1", true], + ["42e03c", "1", false], ["4d001f", "1", false], ["64001f", "1", false], ["42001f", "1", false], ["42e11f", "1", false], ["42e035", "1", false], ["42e03c", "1", false], ["42e01f=invalid", "1", false], diff --git a/tests/quality.test.ts b/tests/quality.test.ts index 29dd21bf6..d8e1aecba 100644 --- a/tests/quality.test.ts +++ b/tests/quality.test.ts @@ -54,11 +54,14 @@ afterEach(() => { describe("realtime quality controls", () => { it("bounds custom size, orientation and frame rate together before capture", () => { - expect(captureFramerateCeiling(QUALITY_RESOLUTIONS["4K"])).toBe(30); - expect(captureFramerateCeiling({ width: 2160, height: 3840 })).toBe(30); + expect(captureFramerateCeiling(QUALITY_RESOLUTIONS["4K"])).toBe(60); + expect(captureFramerateCeiling({ width: 2160, height: 3840 })).toBe(60); expect(captureFramerateCeiling(QUALITY_RESOLUTIONS["1440p"])).toBe(60); expect(captureFramerateCeiling({ width: 1440, height: 2560 })).toBe(60); - expect(captureFramerateCeiling({ width: 3840, height: 2100 })).toBe(31); + expect(captureFramerateCeiling({ width: 3840, height: 2100 })).toBe(60); + const level51 = videoCaptureLimits({ maxVideoMacroblocksPerSecond: 983_040 }); + expect(captureFramerateCeiling(QUALITY_RESOLUTIONS["4K"], level51)).toBe(30); + expect(captureFramerateCeiling({ width: 2160, height: 3840 }, level51)).toBe(30); for (const size of [{ width: 3840, height: 3840 }, { width: 1921, height: 1080 }, { width: 4.5, height: 4 }, { width: Infinity, height: 4 }, { width: 2, height: 4 }]) { expect(captureFramerateCeiling(size)).toBe(0); @@ -91,6 +94,26 @@ describe("realtime quality controls", () => { expect(videoTrack.applyConstraints).not.toHaveBeenCalled(); }); + it("requests 4K60 in either orientation through the existing capture controls", async () => { + const { stream, videoTrack } = createVideoStream(); + const getDisplayMedia = vi.fn(async () => stream); + vi.stubGlobal("navigator", { mediaDevices: { getDisplayMedia } }); + for (const resolution of [QUALITY_RESOLUTIONS["4K"], { width: 2160, height: 3840 }]) { + const profile = { ...QUALITY_PROFILES["1080p60"], resolution }; + expect(supportsCaptureProfile(profile)).toBe(true); + await captureDisplay(profile); + expect(getDisplayMedia).toHaveBeenLastCalledWith(expect.objectContaining({ + video: expect.objectContaining({ + width: expect.objectContaining({ max: resolution.width }), + height: expect.objectContaining({ max: resolution.height }), + frameRate: expect.objectContaining({ max: 60 }), + }), + })); + await applyCaptureProfile(stream, profile); + expect(videoTrack.applyConstraints).toHaveBeenCalled(); + } + }); + it("keeps exactly three recommended profiles with 1080p30 as the default", () => { expect(Object.keys(QUALITY_PROFILES)).toEqual([ "720p30", From f7c607edc5a7266836a101748d806f91026dd7ba Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:18:36 +0800 Subject: [PATCH 153/262] docs(media): record operating range validation gaps --- docs/research/native-client-media.md | 22 ++++++++++++++++++++++ docs/todo.md | 7 +++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index c68e01a7d..26be806fc 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -745,6 +745,28 @@ may exceed local support, but cannot authorize larger local allocations. This separation also appears in [mediasoup RTP parameters and capabilities](https://mediasoup.org/documentation/v3/mediasoup/rtp-parameters-and-capabilities/). +The operating-range review on 2026-10-11 found that the current 15 fps / 2 Mbps +source minimum and 12 Mbps maximum retain the earlier menu range; they are not +codec limits. [LiveKit's screen-share presets](https://github.com/livekit/client-sdk-js/blob/main/src/room/track/options.ts) +include 720p5 at 800 kbps. [YouTube's H.264 live-ingest guidance](https://support.google.com/youtube/answer/2853702?hl=en) +recommends 50 Mbps for 4K60, but its ingest/re-encoding workload is not a Piik +WebRTC operating policy. These references motivate testing both ends of the +range, rather than inheriting the old controls or copying another service's cap. + +An offline Windows control used the production H.264 encoder and retained-surface +decoder with synthetic frames. It requested 720p5 at 800 kbps, 720p15 at 1 Mbps, +and 4K60 at 40 and 50 Mbps. Every case was rejected by product profile admission; +the codec-only path encoded and decoded all 30 frames per case. This establishes +local codec feasibility, not sustained bitrate/quality, complete-route support +or a new accepted control range. The ignored control and output are in +`build/candidate-review/four-k60/operating-range.cpp` and `operating-range.jsonl`. + +Before widening source controls, check the existing Browser SFU and Native +lower-output construction together: both use a quarter bitrate with a 150 kbps +floor. A lower source setting must not produce a derived output whose ceiling +exceeds that source. Preserve frame-byte, allocation, output-count and receiver +limits; do not replace the old menu ceiling with unbounded resource permission. + [WebRTC statistics](https://www.w3.org/TR/webrtc-stats/#dom-rtcicecandidatepairstats-availableoutgoingbitrate) describe estimated available capacity, not a configured send-rate ceiling; [Pion exposes it as a floating-point observation](https://github.com/pion/ice/blob/main/stats.go). diff --git a/docs/todo.md b/docs/todo.md index 385dc8cd4..29e011f29 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -75,8 +75,11 @@ Implementation needs a concrete useful result within the product model. actual output. Numeric request, capability and observation boundaries are now separate from implementation admission. Keep receiver, codec/alignment and resource budgets intact; opening an output range still needs end-to-end evidence. - Retain the current audio sender ceiling and bounded video controls; extending - their operating range needs a concrete use case and end-to-end evidence. + Review the inherited video operating range: the 15 fps / 2 Mbps source minimum + prevents low-rate source requests, while 12 Mbps can constrain 4K60 motion quality. + The assessment now has local codec controls beyond both ends; widening the + controls still needs complete-route evidence and coherent lower-output limits. + Keep the audio sender ceiling backed by the existing packet-size evidence. Numeric audio and video controls are implemented; the assessment owns local capture, codec, derived-output, SFU and bounded mixed-route evidence. Remaining receiver acceptance needs physical limited decoders and camera From 75e48d8c74b832d61d82188575339bf10518f4fa Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:53:57 +0800 Subject: [PATCH 154/262] feat(media): broaden bounded video controls --- docs/research/native-client-media.md | 39 +++++--- docs/standards/media-quality.md | 9 +- docs/status.md | 4 +- docs/todo.md | 13 +-- internal/app/nativecapture/protocol_test.go | 10 +- internal/app/nativecapture/stream.go | 7 +- internal/app/nativecontrol/session_test.go | 4 +- native/capture/darwin/main.swift | 5 +- native/capture/linux/main.c | 6 +- native/capture/linux/main.test.c | 13 ++- native/capture/windows/main.cpp | 8 +- native/capture/windows/output_worker.test.cpp | 18 ++++ .../components/living/QualityPresets.tsx | 94 ++++++++++++++----- src/client/media/quality.ts | 6 +- src/client/pages/ControlsPreview.tsx | 21 +++-- src/client/pages/HostPage.tsx | 66 +------------ src/client/sfu/publisher.ts | 5 +- src/client/styles.css | 15 ++- src/client/webrtc/video-codec-preflight.ts | 5 +- tests/quality.test.ts | 19 +++- tests/sfu-client.test.ts | 17 ++++ tests/video-codec-preflight.test.ts | 10 ++ 22 files changed, 239 insertions(+), 155 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 26be806fc..f37dc5e60 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -745,27 +745,36 @@ may exceed local support, but cannot authorize larger local allocations. This separation also appears in [mediasoup RTP parameters and capabilities](https://mediasoup.org/documentation/v3/mediasoup/rtp-parameters-and-capabilities/). -The operating-range review on 2026-10-11 found that the current 15 fps / 2 Mbps -source minimum and 12 Mbps maximum retain the earlier menu range; they are not +The operating-range review found that the inherited 15 fps / 2 Mbps +source minimum and 12 Mbps maximum retained the earlier menu range; they were not codec limits. [LiveKit's screen-share presets](https://github.com/livekit/client-sdk-js/blob/main/src/room/track/options.ts) include 720p5 at 800 kbps. [YouTube's H.264 live-ingest guidance](https://support.google.com/youtube/answer/2853702?hl=en) recommends 50 Mbps for 4K60, but its ingest/re-encoding workload is not a Piik WebRTC operating policy. These references motivate testing both ends of the range, rather than inheriting the old controls or copying another service's cap. -An offline Windows control used the production H.264 encoder and retained-surface -decoder with synthetic frames. It requested 720p5 at 800 kbps, 720p15 at 1 Mbps, -and 4K60 at 40 and 50 Mbps. Every case was rejected by product profile admission; -the codec-only path encoded and decoded all 30 frames per case. This establishes -local codec feasibility, not sustained bitrate/quality, complete-route support -or a new accepted control range. The ignored control and output are in -`build/candidate-review/four-k60/operating-range.cpp` and `operating-range.jsonl`. - -Before widening source controls, check the existing Browser SFU and Native -lower-output construction together: both use a quarter bitrate with a 150 kbps -floor. A lower source setting must not produce a derived output whose ceiling -exceeds that source. Preserve frame-byte, allocation, output-count and receiver -limits; do not replace the old menu ceiling with unbounded resource permission. +The candidate now admits 1..60 integer fps and 0.1..50 Mbps. Its lower-output +construction caps the usual 150 kbps floor by the source ceiling, in both Browser +SFU and Native. Browser and Native preflight require positive encoded progress; +the old one-frame tolerance could otherwise pass zero output at 1 fps. +An earlier raw H.264 encode/decode control established local feasibility beyond +the old range. Production Windows output-worker checks now cover H.264/VP8 at +720p1/100 kbps, 720p5/800 kbps and 4K60/50 Mbps, including source-size replacement. +Native encoded input also produced the expected half-size output at the low and +high bounds. Each derivation retained its existing input-format generation; +source changes still retire and replan that optional output through its owner. +Browser loopback with the real sender controls also delivered decoded progress +at all three settings and after replacing the 1 fps source. The 1 fps H.264 +preflight passed; the 4K60 preflight selected VP8 on this machine even though +explicit H.264 delivered frames. Admission is not a sustained-cadence guarantee. + +A uniform-noise 4K1/50 Mbps stress input reached the retained 4 MiB access-unit +guard in both Native codecs; VP8 produced a 6,199,837-byte frame. The output was +rejected rather than expanding allocation limits. Large bitrate/low-cadence +requests are not permission for unbounded frames. These local controls establish +useful operation and bounded failure, not sustained field quality or complete-route +performance. Ignored controls and results are under +`build/candidate-review/four-k60/` (`range-worker.cpp`, `range-browser.js`). [WebRTC statistics](https://www.w3.org/TR/webrtc-stats/#dom-rtcicecandidatepairstats-availableoutgoingbitrate) describe estimated available capacity, not a configured send-rate ceiling; diff --git a/docs/standards/media-quality.md b/docs/standards/media-quality.md index e04f7d490..4289aeffe 100644 --- a/docs/standards/media-quality.md +++ b/docs/standards/media-quality.md @@ -138,13 +138,18 @@ rate in the same quality operation when the larger size requires a lower limit. Larger wire-valid requests do not authorize an adapter to exceed its local capture, codec or allocation limits. Numeric requests use representational bounds rather than the current control -ranges. Current capture still admits integer 15..60 fps and 2..12 Mbps, further -limited by picture size. These are current product controls, not universal codec +ranges. Candidate capture admits integer 1..60 fps and 0.1..50 Mbps, further +limited by picture size. Advanced controls accept typed values alongside sliders; +the three recommended profiles and default remain unchanged. These are useful +product controls, not universal codec limits or a minimum for framework-driven output. Codec and resource budgets apply separately. App descriptions may advertise stronger capabilities; controls use their intersection with local support. An unsupported request is an operation failure, not malformed signaling. An unsupported saved preference does not seed capture or overwrite its stored value with a fallback. +Derived outputs must stay within their source bitrate ceiling, including when +that ceiling is below a usual lower-output floor. Codec preflight must observe +real encoded progress; cadence tolerance alone cannot make zero output pass. Frame rate, bitrate, and `maintain-resolution | balanced | maintain-framerate` retain their controls. Display video uses the standard `contentHint = "motion"` for game motion. diff --git a/docs/status.md b/docs/status.md index a277813b8..a16f778ef 100644 --- a/docs/status.md +++ b/docs/status.md @@ -67,7 +67,9 @@ capture v7 and SQLite schema 2 remain unchanged. Matched candidate App/Web/Server builds are required; 1.x participants are rejected by the existing compatibility checks. Custom width/height controls apply combined size and frame-rate limits, admitting -4K60. Windows and Linux output preserve source shape within the selected ceiling. +4K60. Advanced video controls now span 1..60 fps and 0.1..50 Mbps while retaining +the three recommended presets. Windows and Linux output preserve source shape +within the selected ceiling. The [media assessment](./research/native-client-media.md#media-capability-extension-assessment) owns local codec, derived-output, SFU, Browser/Native fanout and live-control evidence. The macOS geometry implementation still needs SDK compilation and diff --git a/docs/todo.md b/docs/todo.md index 29e011f29..534fda8ad 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -27,8 +27,8 @@ Git/PRs own completed history. [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s 256/320 kbps request and [#469](https://github.com/TNTcraftHIM/Piik/pull/469)'s 4K request overlap the numeric-media candidate; neither PR has been merged. 4K60 is included in - candidate admission and still needs route/device acceptance; 40 Mbps needs - separate evidence and is not accepted scope. Preserve + candidate admission, alongside the tested wider video control range, and still + needs route/device acceptance. Preserve contributor credit and explain the disposition at integration. Rewrite [#459](https://github.com/TNTcraftHIM/Piik/pull/459)'s earlier maintenance scope and compatibility claims around the final 2.0 result before integration. @@ -75,10 +75,11 @@ Implementation needs a concrete useful result within the product model. actual output. Numeric request, capability and observation boundaries are now separate from implementation admission. Keep receiver, codec/alignment and resource budgets intact; opening an output range still needs end-to-end evidence. - Review the inherited video operating range: the 15 fps / 2 Mbps source minimum - prevents low-rate source requests, while 12 Mbps can constrain 4K60 motion quality. - The assessment now has local codec controls beyond both ends; widening the - controls still needs complete-route evidence and coherent lower-output limits. + The inherited menu-only source range is replaced by 1..60 fps / 0.1..50 Mbps, + with coherent lower-output bounds and actual-progress preflight. Local + production-codec, source-change and Browser loopback checks pass; full-route + acceptance must include the low-rate settings as well as 4K60. The Linux native + build and range tests remain pending. Keep the audio sender ceiling backed by the existing packet-size evidence. Numeric audio and video controls are implemented; the assessment owns local capture, codec, derived-output, SFU and bounded mixed-route evidence. diff --git a/internal/app/nativecapture/protocol_test.go b/internal/app/nativecapture/protocol_test.go index 1e7806a3d..4774503ea 100644 --- a/internal/app/nativecapture/protocol_test.go +++ b/internal/app/nativecapture/protocol_test.go @@ -286,6 +286,9 @@ func TestCaptureTargetIdentityAndAudioScope(t *testing.T) { func TestVideoProfileUsesTheProductBounds(t *testing.T) { for _, profile := range []VideoProfile{ + {Width: 1280, Height: 720, Framerate: 1, Bitrate: 100_000, Preference: "balanced"}, + {Width: 1280, Height: 720, Framerate: 5, Bitrate: 800_000, Preference: "balanced"}, + {Width: 3840, Height: 2160, Framerate: 60, Bitrate: 50_000_000, Preference: "balanced"}, {Width: 854, Height: 480, Framerate: 15, Bitrate: 2_000_000, Preference: "maintain-resolution"}, {Width: 1280, Height: 720, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"}, {Width: 1920, Height: 1080, Framerate: 60, Bitrate: 8_000_000, Preference: "maintain-framerate"}, @@ -303,7 +306,7 @@ func TestVideoProfileUsesTheProductBounds(t *testing.T) { t.Fatalf("valid profile rejected: %+v", profile) } for _, output := range ScreenShareOutputs(profile) { - if !output.Valid() { + if !output.Valid() || output.Bitrate > profile.Bitrate { t.Fatalf("valid source profile produced an invalid output: %+v", output) } } @@ -321,8 +324,9 @@ func TestVideoProfileUsesTheProductBounds(t *testing.T) { {Width: 3840, Height: 2160, Framerate: 61, Bitrate: 3_000_000, Preference: "balanced"}, {Width: 2160, Height: 3840, Framerate: 120, Bitrate: 3_000_000, Preference: "balanced"}, {Width: ^uint32(0), Height: ^uint32(0), Framerate: ^uint32(0), Bitrate: 3_000_000, Preference: "balanced"}, - {Width: 1920, Height: 1080, Framerate: 14, Bitrate: 5_000_000, Preference: "balanced"}, - {Width: 1920, Height: 1080, Framerate: 30, Bitrate: 12_000_001, Preference: "balanced"}, + {Width: 1920, Height: 1080, Framerate: 0, Bitrate: 5_000_000, Preference: "balanced"}, + {Width: 1920, Height: 1080, Framerate: 30, Bitrate: 99_999, Preference: "balanced"}, + {Width: 1920, Height: 1080, Framerate: 30, Bitrate: 50_000_001, Preference: "balanced"}, {Width: 1920, Height: 1080, Framerate: 30, Bitrate: 5_000_000, Preference: "unknown"}, } { if profile.Valid() { diff --git a/internal/app/nativecapture/stream.go b/internal/app/nativecapture/stream.go index 5490bccb6..82d8962c9 100644 --- a/internal/app/nativecapture/stream.go +++ b/internal/app/nativecapture/stream.go @@ -24,7 +24,7 @@ const ( maxPreviewBytes = 192 * 1024 maxOutputs = 6 minOutputBitrate = 1_000 // Codec rate APIs use whole kbps. - MaxVideoBitrate = 12_000_000 + MaxVideoBitrate = 50_000_000 maxSources = 1024 maxSourceTitleBytes = 4096 // Leave room for the control envelope within loopback's 256 KiB message. @@ -98,7 +98,7 @@ func ScreenShareOutputs(profile VideoProfile) []OutputProfile { return []OutputProfile{ { Width: profile.Width / 4 * 2, Height: profile.Height / 4 * 2, - Framerate: profile.Framerate, Bitrate: max(150_000, profile.Bitrate/4), + Framerate: profile.Framerate, Bitrate: min(profile.Bitrate, max(150_000, profile.Bitrate/4)), }, { Width: profile.Width, Height: profile.Height, @@ -111,8 +111,7 @@ func (profile VideoProfile) Valid() bool { // The source ceiling must also yield a valid half-size output. Preset labels // belong to the UI, not the capture process's numeric resource boundary. return ValidVideoFormat(profile.Width, profile.Height, profile.Framerate) && profile.Width >= 4 && profile.Height >= 4 && - profile.Framerate >= 15 && - profile.Bitrate >= 2_000_000 && profile.Bitrate <= MaxVideoBitrate && + profile.Bitrate >= 100_000 && profile.Bitrate <= MaxVideoBitrate && validVideoPreference(profile.Preference) } diff --git a/internal/app/nativecontrol/session_test.go b/internal/app/nativecontrol/session_test.go index 8cdc4969c..aebceff7a 100644 --- a/internal/app/nativecontrol/session_test.go +++ b/internal/app/nativecontrol/session_test.go @@ -699,8 +699,8 @@ func TestUpdateShareRequiresTheCurrentStrictProfile(t *testing.T) { for _, replace := range [][2]string{ {`"width":1920`, `"width":32768`}, {`"width":1920`, `"width":1921`}, {`"maxFramerate":30`, `"maxFramerate":120`}, {`"maxFramerate":30`, `"maxFramerate":30.5`}, - {`"maxFramerate":30`, `"maxFramerate":4294967326`}, {`"maxFramerate":30`, `"maxFramerate":1`}, - {`"maxBitrate":5000000`, `"maxBitrate":40000000`}, {`"maxBitrate":5000000`, `"maxBitrate":1000`}, + {`"maxFramerate":30`, `"maxFramerate":4294967326`}, {`"maxFramerate":30`, `"maxFramerate":0.5`}, + {`"maxBitrate":5000000`, `"maxBitrate":50000001`}, {`"maxBitrate":5000000`, `"maxBitrate":1000`}, } { assertOperationFailure(t, session, strings.Replace(valid, replace[0], replace[1], 1)) } diff --git a/native/capture/darwin/main.swift b/native/capture/darwin/main.swift index 33d5975d9..3bdc12036 100644 --- a/native/capture/darwin/main.swift +++ b/native/capture/darwin/main.swift @@ -1517,8 +1517,7 @@ private func videoProfile(_ arguments: [String]) throws -> VideoProfile { throw CaptureFailure(description: "invalid video profile arguments") } guard validVideoFormat(width, height, frameRate), width >= 4, height >= 4, - frameRate >= 15, - bitrate >= 2_000_000, bitrate <= 12_000_000 else { + bitrate >= 100_000, bitrate <= 50_000_000 else { throw CaptureFailure(description: "video profile is outside the product bounds") } return VideoProfile( @@ -1546,7 +1545,7 @@ private func outputProfiles(_ arguments: [String], start: Int = 23, validVideoFormat(width, height, fps), width <= (source?.width ?? maxVideoDimension), height <= (source?.height ?? maxVideoDimension), fps <= (source?.frameRate ?? 60), - bitrate >= 1000, bitrate <= (source?.bitrate ?? 12_000_000) else { + bitrate >= 1000, bitrate <= (source?.bitrate ?? 50_000_000) else { throw CaptureFailure(description: "output profile is outside source bounds") } result.append(OutputProfile(width: width, height: height, fps: fps, bitrate: bitrate)) diff --git a/native/capture/linux/main.c b/native/capture/linux/main.c index f3e06da4b..57cdb9ae1 100644 --- a/native/capture/linux/main.c +++ b/native/capture/linux/main.c @@ -402,7 +402,7 @@ static gboolean parse_outputs(int count, char **values, guint start, CaptureRun !parse_uint(values[offset + 1], 2, kMaxVideoDimension, &output->profile.width) || !parse_uint(values[offset + 2], 2, kMaxVideoDimension, &output->profile.height) || !parse_uint(values[offset + 3], 1, 60, &output->profile.frame_rate) || - !parse_uint(values[offset + 4], 1000, 12000000, &output->profile.bit_rate) || + !parse_uint(values[offset + 4], 1000, 50000000, &output->profile.bit_rate) || !valid_video_format(output->profile.width, output->profile.height, output->profile.frame_rate)) return FALSE; } @@ -454,8 +454,8 @@ static gboolean parse_profile(int count, char **values, CaptureRun *run, !parse_uint(values[9], 0, 63, encoder_index) || !parse_uint(values[11], 1, 16384, &profile->width) || !parse_uint(values[13], 1, 16384, &profile->height) || - !parse_uint(values[15], 15, 60, &profile->frame_rate) || - !parse_uint(values[17], 2000000, 12000000, &profile->bit_rate)) { + !parse_uint(values[15], 1, 60, &profile->frame_rate) || + !parse_uint(values[17], 100000, 50000000, &profile->bit_rate)) { return FALSE; } if (!valid_video_format(profile->width, profile->height, profile->frame_rate) || diff --git a/native/capture/linux/main.test.c b/native/capture/linux/main.test.c index e2f3548ac..48c541c5a 100644 --- a/native/capture/linux/main.test.c +++ b/native/capture/linux/main.test.c @@ -31,7 +31,7 @@ static void fixture_encoder_class_init(FixtureEncoderClass *klass) { object_class->set_property = fixture_set_property; object_class->get_property = fixture_get_property; g_object_class_install_property(object_class, 1, - g_param_spec_uint("bitrate", "Bitrate", "Bitrate in bps", 1000, 12000000, + g_param_spec_uint("bitrate", "Bitrate", "Bitrate in bps", 1000, 50000000, 3000000, G_PARAM_READWRITE | GST_PARAM_MUTABLE_PLAYING)); } @@ -177,9 +177,18 @@ static void check_slot_profiles(void) { numeric[13] = numeric[25] = "2160"; numeric[15] = numeric[26] = "60"; run = (CaptureRun){0}; - g_assert_false(parse_profile(G_N_ELEMENTS(numeric), numeric, &run, &encoder)); + g_assert_true(parse_profile(G_N_ELEMENTS(numeric), numeric, &run, &encoder)); numeric[15] = "30"; // Reject an excessive derived output independently. g_assert_false(parse_profile(G_N_ELEMENTS(numeric), numeric, &run, &encoder)); + const struct { char *fps; char *bitrate; gboolean valid; } rates[] = { + {"1", "100000", TRUE}, {"5", "800000", TRUE}, {"60", "50000000", TRUE}, + {"0", "100000", FALSE}, {"1", "99999", FALSE}, {"60", "50000001", FALSE}}; + for (guint index = 0; index < G_N_ELEMENTS(rates); ++index) { + numeric[15] = numeric[26] = rates[index].fps; + numeric[17] = numeric[27] = rates[index].bitrate; + run = (CaptureRun){0}; + g_assert_cmpint(parse_profile(G_N_ELEMENTS(numeric), numeric, &run, &encoder), ==, rates[index].valid); + } g_assert_false(valid_video_format(G_MAXUINT, G_MAXUINT, G_MAXUINT)); } diff --git a/native/capture/windows/main.cpp b/native/capture/windows/main.cpp index 5183bd502..e834e6e02 100644 --- a/native/capture/windows/main.cpp +++ b/native/capture/windows/main.cpp @@ -967,8 +967,8 @@ DegradationPreference ParseDegradationPreference(const wchar_t* value) { void ValidateVideoProfile(const VideoProfile& profile) { if (!piik::capture::ValidVideoFormat(profile.width, profile.height, profile.frame_rate) || - profile.width < 4 || profile.height < 4 || profile.frame_rate < 15 || - profile.bit_rate < 2'000'000 || profile.bit_rate > 12'000'000) { + profile.width < 4 || profile.height < 4 || + profile.bit_rate < 100'000 || profile.bit_rate > 50'000'000) { Fail("argument-profile", "video profile is outside the product bounds"); } } @@ -982,7 +982,7 @@ void ParseOutputProfiles(ProductArguments& arguments, int first, int count, wcha output.frame_rate = ParseIndex(values[index + 3], "argument-output-fps"); output.bit_rate = ParseIndex(values[index + 4], "argument-output-bitrate"); if (!piik::capture::ValidVideoFormat(output.width, output.height, output.frame_rate) || - output.bit_rate < 1'000 || output.bit_rate > 12'000'000) { + output.bit_rate < 1'000 || output.bit_rate > 50'000'000) { Fail("argument-output", "output profile exceeds codec bounds"); } if (arguments.mode == ProductArguments::Mode::video && @@ -1277,7 +1277,7 @@ bool SustainsEncodedCadence(const AdaptiveEncoder::Factory& create, ID3D11Device if (frame >= warmup_frames && EncoderClock::now() > due + interval * lag) return false; if (frame >= warmup_frames && output && !output->access_unit.bytes.empty()) ++delivered; } - return delivered + lag >= sample_frames; + return delivered > 0 && delivered + lag >= sample_frames; } struct VideoEncoderSelection final { diff --git a/native/capture/windows/output_worker.test.cpp b/native/capture/windows/output_worker.test.cpp index 1c8f62a16..b2e6e2e8c 100644 --- a/native/capture/windows/output_worker.test.cpp +++ b/native/capture/windows/output_worker.test.cpp @@ -268,6 +268,9 @@ void CheckAutoCadenceProbe(ID3D11Device* device) { assert(probe(std::chrono::milliseconds(220), false, deadline, true)); // Sustained slow encoding still fails after warmup. assert(!probe(std::chrono::milliseconds(300), false, deadline)); + profile.frame_rate = 1; + assert(!probe(std::chrono::milliseconds(0), true, deadline)); + assert(probe(std::chrono::milliseconds(0), false, deadline)); bool expired = false; try { probe(std::chrono::milliseconds(0), false, EncoderClock::time_point::min()); @@ -400,6 +403,21 @@ void CheckDecodedDisplayArea(ID3D11Device* device) { int main() { VideoProfile profile{2560, 1440, 60, 12'000'000}; ValidateVideoProfile(profile); + for (const auto& requested : {VideoProfile{1280, 720, 1, 100'000}, + VideoProfile{1280, 720, 5, 800'000}, + VideoProfile{3840, 2160, 60, 50'000'000}}) { + ValidateVideoProfile(requested); + } + for (const auto& requested : {VideoProfile{1280, 720, 0, 100'000}, + VideoProfile{1280, 720, 1, 99'999}, + VideoProfile{3840, 2160, 60, 50'000'001}}) { + try { + ValidateVideoProfile(requested); + assert(false); + } catch (const GateFailure& error) { + assert(error.stage() == "argument-profile"); + } + } for (const SIZE size : {SIZE{1920, 1200}, SIZE{720, 1280}, SIZE{4, 4}, SIZE{3840, 2160}, SIZE{2160, 3840}, SIZE{3000, 2764}}) { ValidateVideoProfile(VideoProfile{static_cast(size.cx), static_cast(size.cy), 60, 3'000'000}); diff --git a/src/client/components/living/QualityPresets.tsx b/src/client/components/living/QualityPresets.tsx index 0d0302231..92679de4c 100644 --- a/src/client/components/living/QualityPresets.tsx +++ b/src/client/components/living/QualityPresets.tsx @@ -2,8 +2,8 @@ import { Fragment, useRef } from "react"; import { MIN_AUDIO_BITRATE } from "../../../shared/protocol"; import { QUALITY_PROFILES, QUALITY_PROFILE_KEYS, QUALITY_RESOLUTIONS, qualityResolutionEqual, - captureFramerateCeiling, supportsCaptureProfile, MIN_VIDEO_FRAMERATE, - VIDEO_CAPTURE_LIMITS, type VideoCaptureLimits, type QualityProfileId, type QualityResolution, + captureFramerateCeiling, supportsCaptureProfile, MIN_VIDEO_FRAMERATE, MIN_VIDEO_BITRATE, + VIDEO_CAPTURE_LIMITS, type VideoCaptureLimits, type QualityProfile, type QualityProfileId, type QualityResolution, SCREEN_AUDIO_BITRATES, type ScreenAudioPreset, } from "../../media/quality"; import { useCopy, type CopyKey } from "../../ui/copy"; @@ -108,6 +108,72 @@ const AUDIO_QUALITY_CAPTIONS: Record = { "very-high": "host.advanced.audio.veryHigh", }; +function QualityNumberInput({ value, min, max, step, unit, name, label, disabled, onChange }: { + value: number; + min: number; + max: number; + step: number; + unit: string; + name: string; + label: string; + disabled: boolean; + onChange: (value: number) => void; +}) { + return ; +} + +export function VideoRateControls({ value, limits, disabled, onChange }: { + value: QualityProfile; + limits: VideoCaptureLimits; + disabled: boolean; + onChange: (value: Partial>) => void; +}) { + const { t, vis } = useCopy(); + const fpsMax = captureFramerateCeiling(value.resolution, limits); + const controls = [ + { name: "maxFramerate", label: "host.advanced.framerate", icon: "frames", hint: "hint-metric-fps", + value: value.maxFramerate, min: MIN_VIDEO_FRAMERATE, max: fpsMax, step: 1, unit: "fps", + change: (maxFramerate: number) => onChange({ maxFramerate }) }, + { name: "maxBitrate", label: "host.advanced.bitrate", icon: "gauge", hint: "hint-metric-bitrate", + value: value.maxBitrate / 1_000_000, min: MIN_VIDEO_BITRATE / 1_000_000, + max: limits.maxVideoBitrate / 1_000_000, step: 0.1, unit: "Mbps", + change: (mbps: number) => onChange({ maxBitrate: Math.round(mbps * 1_000_000) }) }, + ] as const; + return
+ {controls.map(control =>
+ + + + control.change(event.currentTarget.valueAsNumber)} /> + + + +
)} +
; +} + export function AudioBitrateControl({ value, max, disabled, onChange }: { value: number; max: number; @@ -130,27 +196,9 @@ export function AudioBitrateControl({ value, max, disabled, onChange }: { - + onChange(kbps * 1_000)} />
; } diff --git a/src/client/media/quality.ts b/src/client/media/quality.ts index 2b87315e1..eff1c782c 100644 --- a/src/client/media/quality.ts +++ b/src/client/media/quality.ts @@ -23,8 +23,8 @@ export type { export type QualityProfile = QualitySettings; export const STARTUP_VIDEO_ENCODED_FRAMES = 5; -export const MIN_VIDEO_FRAMERATE = 15; -export const MIN_VIDEO_BITRATE = 2_000_000; +export const MIN_VIDEO_FRAMERATE = 1; +export const MIN_VIDEO_BITRATE = 100_000; export interface VideoCaptureLimits { maxVideoResolution: QualityResolution; maxVideoPixels: number; @@ -40,7 +40,7 @@ export const VIDEO_CAPTURE_LIMITS: VideoCaptureLimits = { maxVideoPixels: 3840 * 2160, maxVideoMacroblocksPerSecond: 2_073_600, maxVideoFramerate: 60, - maxVideoBitrate: 12_000_000, + maxVideoBitrate: 50_000_000, }; export function videoCaptureLimits(advertised?: Partial): VideoCaptureLimits { diff --git a/src/client/pages/ControlsPreview.tsx b/src/client/pages/ControlsPreview.tsx index 88380015d..a603c0286 100644 --- a/src/client/pages/ControlsPreview.tsx +++ b/src/client/pages/ControlsPreview.tsx @@ -19,9 +19,9 @@ import { LauncherForm, type AppMode } from "../components/living/LauncherForm"; import { AppReleaseNotice } from "../components/living/AppReleaseNotice"; import { useTheaterMode } from "../components/living/use-theater-mode"; -import { AudioBitrateControl, QualityPresets, ResolutionControl } from "../components/living/QualityPresets"; +import { AudioBitrateControl, QualityPresets, ResolutionControl, VideoRateControls } from "../components/living/QualityPresets"; import { RoomCodeInput } from "../components/living/RoomCodeInput"; -import { MAX_SCREEN_AUDIO_BITRATE, VIDEO_CAPTURE_LIMITS, QUALITY_PROFILES, type QualityProfileId, type QualityResolution } from "../media/quality"; +import { MAX_SCREEN_AUDIO_BITRATE, VIDEO_CAPTURE_LIMITS, QUALITY_PROFILES, matchingQualityProfileId, type QualityProfile } from "../media/quality"; import type { loadCameraPreviews } from "../media/camera-previews"; import { Tooltip } from "../components/living/Tooltip"; import { StatusIndicator } from "../components/living/StatusIndicator"; @@ -57,8 +57,7 @@ export function ControlsPreview() { const [voiceProcessing, setVoiceProcessing] = useState(true); const [microphoneDevice, setMicrophoneDevice] = useState(""); const [sharingSettings, setSharingSettings] = useState(false); - const [resolution, setResolution] = useState(QUALITY_PROFILES["1080p30"].resolution); - const [audioBitrate, setAudioBitrate] = useState(QUALITY_PROFILES["1080p30"].maxAudioBitrate); + const [quality, setQuality] = useState(QUALITY_PROFILES["1080p30"]); const [launchMode, setLaunchMode] = useState("link"); const [majorUpdate, setMajorUpdate] = useState(true); const releaseVersion = majorUpdate ? "v2.0.0" : "v1.1.0"; @@ -66,7 +65,6 @@ export function ControlsPreview() { const [launchSite, setLaunchSite] = useState(""); const [launchPassword, setLaunchPassword] = useState(""); const [launchLan, setLaunchLan] = useState("192.0.2.10"); - const [preset, setPreset] = useState("1080p30"); const [policy, setPolicy] = useState<"open" | "private">("open"); const [roomPassword, setRoomPassword] = useState(false); const [includeCredential, setIncludeCredential] = useState(true); @@ -119,9 +117,12 @@ export function ControlsPreview() {
{ setPreset(id); setResolution(QUALITY_PROFILES[id].resolution); }} />} - picture={ { setResolution(value); setPreset(null); }} />} + presets={ setQuality(current => ({ ...QUALITY_PROFILES[id], maxAudioBitrate: current.maxAudioBitrate }))} />} + picture={<> setQuality(current => ({ ...current, resolution }))} /> + setQuality(current => ({ ...current, ...change }))} />} audio={<>
@@ -129,8 +130,8 @@ export function ControlsPreview() {
- } + setQuality(current => ({ ...current, maxAudioBitrate }))} />} />
diff --git a/src/client/pages/HostPage.tsx b/src/client/pages/HostPage.tsx index 900d69580..d3f9d4152 100644 --- a/src/client/pages/HostPage.tsx +++ b/src/client/pages/HostPage.tsx @@ -52,7 +52,7 @@ import type { ComicKind } from "../components/living/Comic"; import type { HintKind } from "../ui/visual-kinds"; import type { ComicTone } from "../components/living/comic-presentation"; import { Tooltip } from "../components/living/Tooltip"; -import { AudioBitrateControl, QualityPresets, ResolutionControl } from "../components/living/QualityPresets"; +import { AudioBitrateControl, QualityPresets, ResolutionControl, VideoRateControls } from "../components/living/QualityPresets"; import { RoomCodeInput, RoomCodeError } from "../components/living/RoomCodeInput"; import { MetricCell } from "../components/living/Metrics"; import { @@ -130,8 +130,6 @@ import { videoCaptureLimits, captureFramerateCeiling, supportsCaptureProfile, - MIN_VIDEO_FRAMERATE, - MIN_VIDEO_BITRATE, qualityResolutionEqual, qualitySettingsEqual, qualitySettingsLabel, @@ -3596,7 +3594,6 @@ export function HostPage({ ? nativeClientRef.current?.health.nativeMedia.maxAudioBitrate ?? MAX_SCREEN_AUDIO_BITRATE : MAX_SCREEN_AUDIO_BITRATE); const videoLimits = videoCaptureLimits(nativeActive ? nativeClientRef.current?.health.nativeMedia : undefined); - const framerateCeiling = captureFramerateCeiling(advancedQuality.resolution, videoLimits); return (
@@ -3850,65 +3847,8 @@ export function HostPage({ maxFramerate: Math.min(advancedQualityRef.current.maxFramerate, captureFramerateCeiling(resolution, videoLimits)), })} />
-
-
- - - - - - - - changeAdvancedQuality({ - maxFramerate: Number(event.target.value), - }) - } - /> - {advancedQuality.maxFramerate} fps - - -
-
- - - - - - - - changeAdvancedQuality({ - maxBitrate: Number(event.target.value), - }) - } - /> - - {(advancedQuality.maxBitrate / 1_000_000).toFixed(1)}{" "} - Mbps - - - -
-
+
= this.layerCount, @@ -454,7 +455,7 @@ export class SfuPublisher { rid: "q", active: this.activeCount > 0, scaleResolutionDownBy: 2 * scale, - maxBitrate: Math.max(150_000, Math.floor(profile.maxBitrate / 4)), + maxBitrate: Math.min(profile.maxBitrate, Math.max(150_000, Math.floor(profile.maxBitrate / 4))), maxFramerate: profile.maxFramerate, }, high, diff --git a/src/client/styles.css b/src/client/styles.css index 7fd85d532..44e00aa6b 100644 --- a/src/client/styles.css +++ b/src/client/styles.css @@ -1176,8 +1176,11 @@ html[data-theme="dark"] .lr-invite-url { background: #1a2634; border-color: #334 .lr-input:focus-within { border-color: var(--action); outline: 3px solid var(--action); outline-offset: 2px; } .lr-input input { min-width: 0; flex: 1; height: 100%; border: 0; outline: 0; background: transparent; font: inherit; font-size: 16px; } .lr-input input[type="password"]:not(:placeholder-shown) { letter-spacing: .25em; } -.lr-input.lr-audio-bitrate { width: 160px; max-width: 100%; font-family: var(--mono); } -.lr-audio-bitrate span { color: var(--muted); font-size: 12px; } +.lr-input.lr-quality-number { width: 160px; max-width: 100%; font-family: var(--mono); } +.lr-quality-number span { color: var(--muted); font-size: 12px; } +.lr-quality-rate { display: flex; align-items: center; gap: 10px; width: 100%; min-width: 0; } +.lr-quality-rate > input { flex: 1; min-width: 0; width: 0; height: 32px; accent-color: var(--action); } +.lr-quality-rate .lr-quality-number { flex: 0 0 132px; min-height: 40px; padding: 6px 10px; } .lr-input.lr-video-size { display: grid; grid-template-columns: minmax(0, 1fr) auto minmax(0, 1fr) auto; gap: 4px; padding-inline: 8px; width: 252px; max-width: 100%; font-family: var(--mono); } .lr-resolution-controls { display: flex; flex-direction: column; gap: 8px; min-width: 0; } .lr-video-size input { width: 100%; padding: 0; text-align: center; appearance: textfield; } @@ -1835,7 +1838,7 @@ html[data-theme="dark"] .lr-switch::after { background: var(--ink); } .lr-sharing-section .lr-door-group { gap: 8px; } .lr-sharing-panel .lr-sharing-section .lr-door-glyph { color: var(--muted); } .lr-sharing-section .lr-door-glyph .lr-cap { white-space: normal; } -.lr-sharing-limits { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 16px; } +.lr-sharing-limits { display: grid; grid-template-columns: 1fr; gap: 16px; } .lr-sharing-panel .lr-slider { width: 100%; } .lr-sharing-panel .lr-slider output { min-width: 9ch; } .lr-sharing-section .lr-tiles { gap: 8px; } @@ -1854,9 +1857,13 @@ html[data-theme="dark"] .lr-switch::after { background: var(--ink); } @container sharing-settings (max-width: 680px) { .lr-sharing-panel { padding: 16px; border-radius: 20px; } .lr-sharing-columns { grid-template-columns: 1fr; gap: 22px; } - .lr-sharing-limits { grid-template-columns: 1fr; } .lr-sharing-section + .lr-sharing-section { padding: 20px 0 0; border-inline-start: 0; border-top: 1px solid var(--wall-2); } } +@container sharing-settings (max-width: 260px) { + .lr-quality-rate { flex-direction: column-reverse; align-items: stretch; gap: 2px; } + .lr-quality-rate > input { flex: auto; width: 100%; margin-inline: 0; } + .lr-quality-rate .lr-quality-number { flex: auto; width: 100%; } +} .lr-host-share-controls { display: flex; flex-wrap: wrap; align-items: center; justify-content: center; gap: 4px; width: fit-content; max-width: 100%; margin: 12px auto 0; padding: 6px; diff --git a/src/client/webrtc/video-codec-preflight.ts b/src/client/webrtc/video-codec-preflight.ts index e7c892027..4c49741d0 100644 --- a/src/client/webrtc/video-codec-preflight.ts +++ b/src/client/webrtc/video-codec-preflight.ts @@ -235,6 +235,7 @@ export function h264ProbeSustainsTarget( const encoded = current.framesEncoded - baseline.framesEncoded; const source = current.sourceFrames - baseline.sourceFrames; return ( + source > 0 && encoded > 0 && source + frameLag >= expectedFrames && encoded + frameLag >= expectedFrames && encoded + frameLag >= source @@ -242,12 +243,12 @@ export function h264ProbeSustainsTarget( } if ( current.encodedFramesPerSecond === null || - current.sourceFramesPerSecond === null || - current.sourceFramesPerSecond <= 0 + current.sourceFramesPerSecond === null ) { return null; } return ( + current.sourceFramesPerSecond > 0 && current.encodedFramesPerSecond > 0 && current.sourceFramesPerSecond + frameLag >= targetFrameRate && current.encodedFramesPerSecond + frameLag >= targetFrameRate && current.encodedFramesPerSecond + frameLag >= diff --git a/tests/quality.test.ts b/tests/quality.test.ts index d8e1aecba..c0b89a5e3 100644 --- a/tests/quality.test.ts +++ b/tests/quality.test.ts @@ -73,7 +73,7 @@ describe("realtime quality controls", () => { expect(captureFramerateCeiling(QUALITY_RESOLUTIONS["720p"], limited)).toBe(60); expect(videoCaptureLimits({ maxVideoResolution: { width: 8192, height: 8192 }, maxVideoPixels: 8192 * 8192, maxVideoMacroblocksPerSecond: 9_000_000, - maxVideoFramerate: 120, maxVideoBitrate: 40_000_000 })).toEqual(videoCaptureLimits()); + maxVideoFramerate: 120, maxVideoBitrate: 100_000_000 })).toEqual(videoCaptureLimits()); const slower = videoCaptureLimits({ maxVideoFramerate: 30, maxVideoBitrate: 5_000_000 }); expect(supportsCaptureProfile(QUALITY_PROFILES["1080p30"], slower)).toBe(true); expect(supportsCaptureProfile(QUALITY_PROFILES["1080p60"], slower)).toBe(false); @@ -83,7 +83,8 @@ describe("realtime quality controls", () => { const { stream, videoTrack } = createVideoStream(); const getDisplayMedia = vi.fn(async () => stream); vi.stubGlobal("navigator", { mediaDevices: { getDisplayMedia } }); - for (const change of [{ maxFramerate: 120 }, { maxFramerate: 30.5 }, { maxBitrate: 40_000_000 }, + for (const change of [{ maxFramerate: 120 }, { maxFramerate: 30.5 }, { maxFramerate: 0 }, + { maxBitrate: 99_999 }, { maxBitrate: 50_000_001 }, { resolution: { width: 32768, height: 32768 } }, { maxAudioBitrate: 510_000 }]) { const requested = { ...QUALITY_PROFILES["1080p30"], ...change }; expect(supportsCaptureProfile(requested)).toBe(false); @@ -94,12 +95,24 @@ describe("realtime quality controls", () => { expect(videoTrack.applyConstraints).not.toHaveBeenCalled(); }); + it("admits useful low-rate and high-rate requests without changing presets", async () => { + const { stream, videoTrack } = createVideoStream(); + for (const [maxFramerate, maxBitrate] of [[1, 100_000], [5, 800_000], [60, 50_000_000]]) { + const profile = { ...QUALITY_PROFILES["1080p30"], maxFramerate, maxBitrate }; + expect(supportsCaptureProfile(profile)).toBe(true); + await applyCaptureProfile(stream, profile); + expect(videoTrack.applyConstraints).toHaveBeenLastCalledWith(expect.objectContaining({ + frameRate: { ideal: maxFramerate, max: maxFramerate }, + })); + } + }); + it("requests 4K60 in either orientation through the existing capture controls", async () => { const { stream, videoTrack } = createVideoStream(); const getDisplayMedia = vi.fn(async () => stream); vi.stubGlobal("navigator", { mediaDevices: { getDisplayMedia } }); for (const resolution of [QUALITY_RESOLUTIONS["4K"], { width: 2160, height: 3840 }]) { - const profile = { ...QUALITY_PROFILES["1080p60"], resolution }; + const profile = { ...QUALITY_PROFILES["1080p60"], resolution, maxBitrate: 50_000_000 }; expect(supportsCaptureProfile(profile)).toBe(true); await captureDisplay(profile); expect(getDisplayMedia).toHaveBeenLastCalledWith(expect.objectContaining({ diff --git a/tests/sfu-client.test.ts b/tests/sfu-client.test.ts index 8d3fffb35..7ef8e9199 100644 --- a/tests/sfu-client.test.ts +++ b/tests/sfu-client.test.ts @@ -244,6 +244,23 @@ function feedVideoStats( } describe("embedded SFU browser transport", () => { + it.each([100_000, 200_000, 50_000_000])("keeps every simulcast output within the %i bps source ceiling", async (maxBitrate) => { + const send = vi.fn(() => true); + const host = new SfuPublisher({ send, onDisconnected: vi.fn() }); + cleanups.push(() => host.disconnect()); + await host.connect(config); + expect(await host.activate(stream(new FakeTrack("video")), + { ...QUALITY_PROFILES["1080p30"], maxBitrate, maxFramerate: 5 }, "h264")).toBe(true); + const encodings = FakePc.instances[0]!.transceivers[0]!.sender.getParameters().encodings; + expect(encodings).toHaveLength(2); + expect(encodings[1]!.maxBitrate).toBe(maxBitrate); + for (const encoding of encodings) { + expect(encoding.maxBitrate).toBeGreaterThan(0); + expect(encoding.maxBitrate).toBeLessThanOrEqual(maxBitrate); + expect(encoding.maxFramerate).toBe(5); + } + }); + it("uses capture dimensions for simulcast before a generated sender has its first frame", async () => { class Generator extends FakeTrack { writable = new WritableStream(); diff --git a/tests/video-codec-preflight.test.ts b/tests/video-codec-preflight.test.ts index 17dbfe305..0b3c75e8a 100644 --- a/tests/video-codec-preflight.test.ts +++ b/tests/video-codec-preflight.test.ts @@ -59,6 +59,16 @@ describe("H264 sender preflight", () => { expect(h264ProbeSustainsTarget(baseline, current, 30)).toBe(false); }); + it("requires real progress even when one frame is within the cadence tolerance", () => { + for (const [source, encoded, expected] of [[0, 0, false], [1, 0, false], [0, 1, false], [1, 1, true]] as const) { + expect(h264ProbeSustainsTarget(sample(), sample({ timestamp: 2_000, + sourceFrames: 10 + source, framesEncoded: 10 + encoded }), 1)).toBe(expected); + expect(h264ProbeSustainsTarget(sample(), sample({ timestamp: 2_000, + sourceFrames: null, framesEncoded: null, + sourceFramesPerSecond: source, encodedFramesPerSecond: encoded }), 1)).toBe(expected); + } + }); + it("waits for enough source progress before deciding", () => { const baseline = sample(); const current = sample({ From 06c6960c8ff6d9261bdcd46c0a423aef5acdd2c0 Mon Sep 17 00:00:00 2001 From: AkazaAkalin <478080766@qq.com> Date: Sat, 10 Oct 2026 19:43:19 +0800 Subject: [PATCH 155/262] fix(linux): select PipeWire node IDs with the path property (cherry picked from commit b10068cd44f7362ae1877f6010c63c74c196dc9c) --- docs/todo.md | 13 ++++++------- native/capture/linux/main.c | 19 +++++++++++++++---- native/capture/linux/main.test.c | 13 +++++++++++++ native/capture/linux/portal.c | 3 --- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/docs/todo.md b/docs/todo.md index 534fda8ad..2aab2404d 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -16,13 +16,12 @@ Git/PRs own completed history. Prepare one coherent candidate and stop before publication. Preserve the existing product model; evaluate held features against measurable benefit and total complexity before accepting a changed contract. -- [ ] **Linux Portal target identity (#470).** Review of the - [contributor fix](https://github.com/TNTcraftHIM/Piik/pull/470) and upstream - PipeWire confirms that the current fallback passes a node ID as an object - serial. Integrate the fix with attribution, retain serial targeting when - available, and use the node-ID selector otherwise. Run the Linux native - checks and verify a Portal that omits `pipewire-serial`; this fix is not yet - in the candidate. +- [ ] **Linux Portal target identity acceptance (#470).** The + [contributor fix](https://github.com/TNTcraftHIM/Piik/pull/470) is integrated + with authorship retained: serial and node-ID targeting remain distinct. + Run the candidate's Linux native checks and verify a Portal that omits + `pipewire-serial`. Upstream source review confirms the property distinction; + contributor device evidence does not replace candidate acceptance. - [ ] **Reconcile open PRs with the final candidate.** [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s 256/320 kbps request and [#469](https://github.com/TNTcraftHIM/Piik/pull/469)'s 4K request overlap the diff --git a/native/capture/linux/main.c b/native/capture/linux/main.c index 57cdb9ae1..01a4fe500 100644 --- a/native/capture/linux/main.c +++ b/native/capture/linux/main.c @@ -1136,6 +1136,13 @@ static void append_video_scale(GString *pipeline, const VideoProfile *profile) { g_string_append(pipeline, "pixel-aspect-ratio=1/1 ! "); } +static char *pipewire_target(const PiikPortalCapture *portal) { + // target-object accepts a name or object serial; path accepts a node ID. + return portal->target_object != NULL + ? g_strdup_printf("target-object=%s", portal->target_object) + : g_strdup_printf("path=%u", portal->node_id); +} + static int capture_video(int count, char **values) { CaptureRun run = {.last_input_timestamp = GST_CLOCK_TIME_NONE}; gboolean encoded = count > 1 && strcmp(values[1], "--encoded-video") == 0; @@ -1168,10 +1175,14 @@ static int capture_video(int count, char **values) { "max-buffers=1 max-bytes=1048576 max-time=0 " "caps=video/x-h264,stream-format=byte-stream,alignment=au ! " "h264parse ! decodebin ! video/x-raw ! tee name=frames "); - } else g_string_append_printf(pipeline_text, - "pipewiresrc name=source fd=%d target-object=%s do-timestamp=true ! " - "videorate drop-only=true ! video/x-raw,framerate=%u/1 ! tee name=frames ", - portal.pipewire_fd, portal.target_object, run.profile.frame_rate); + } else { + char *target = pipewire_target(&portal); + g_string_append_printf(pipeline_text, + "pipewiresrc name=source fd=%d %s do-timestamp=true ! " + "videorate drop-only=true ! video/x-raw,framerate=%u/1 ! tee name=frames ", + portal.pipewire_fd, target, run.profile.frame_rate); + g_free(target); + } for (guint index = 0; index < run.output_count; ++index) { const VideoProfile *profile = &run.outputs[index].profile; g_string_append_printf(pipeline_text, diff --git a/native/capture/linux/main.test.c b/native/capture/linux/main.test.c index 48c541c5a..8ea905433 100644 --- a/native/capture/linux/main.test.c +++ b/native/capture/linux/main.test.c @@ -548,8 +548,21 @@ static void check_output_failure_isolation(gboolean post_error) { g_mutex_clear(&run->lock); } +static void check_pipewire_target_identity(void) { + PiikPortalCapture portal = {.node_id = 42}; + char *target = pipewire_target(&portal); + g_assert_cmpstr(target, ==, "path=42"); + g_free(target); + + portal.target_object = "9001"; + target = pipewire_target(&portal); + g_assert_cmpstr(target, ==, "target-object=9001"); + g_free(target); +} + int main(int argc, char **argv) { gst_init(&argc, &argv); + check_pipewire_target_identity(); check_encoder_admission_and_control(); check_bus_error_retains_primary_cause(); check_slot_profiles(); diff --git a/native/capture/linux/portal.c b/native/capture/linux/portal.c index 586e8cd8f..4defa3bf4 100644 --- a/native/capture/linux/portal.c +++ b/native/capture/linux/portal.c @@ -125,9 +125,6 @@ gboolean piik_portal_capture_open(const char *restore_token, piik_portal_capture_close(capture); return FALSE; } - if (capture->target_object == NULL) { - capture->target_object = g_strdup_printf("%u", capture->node_id); - } capture->restore_token = xdp_session_get_restore_token(capture->session); return TRUE; } From 22137ef82930e954013db6886dacb546e4487de4 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:02:08 +0800 Subject: [PATCH 156/262] docs(media): record low-rate sender acceptance --- docs/research/native-client-media.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index f37dc5e60..d14b39ec6 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -764,7 +764,10 @@ Native encoded input also produced the expected half-size output at the low and high bounds. Each derivation retained its existing input-format generation; source changes still retire and replan that optional output through its owner. Browser loopback with the real sender controls also delivered decoded progress -at all three settings and after replacing the 1 fps source. The 1 fps H.264 +at all three settings and after replacing the 1 fps source. At 1 fps / 100 kbps, +the production Browser HostPeer also delivered to two local receivers through +ordinary senders and the shared encoding pool, for both H.264 and VP8. This +bounded check does not cover room routing or physical-network recovery. The 1 fps H.264 preflight passed; the 4K60 preflight selected VP8 on this machine even though explicit H.264 delivered frames. Admission is not a sustained-cadence guarantee. @@ -774,7 +777,8 @@ rejected rather than expanding allocation limits. Large bitrate/low-cadence requests are not permission for unbounded frames. These local controls establish useful operation and bounded failure, not sustained field quality or complete-route performance. Ignored controls and results are under -`build/candidate-review/four-k60/` (`range-worker.cpp`, `range-browser.js`). +`build/candidate-review/four-k60/` (`range-worker.cpp`, `range-browser.js`, +`low-rate-pool.js`). [WebRTC statistics](https://www.w3.org/TR/webrtc-stats/#dom-rtcicecandidatepairstats-availableoutgoingbitrate) describe estimated available capacity, not a configured send-rate ceiling; From a7a47ce44d7f329b2bdb357305ec6fa3f0507439 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:18:44 +0800 Subject: [PATCH 157/262] docs(media): replace stale capture contract details --- docs/research/node-local-media-design.md | 42 +++++++++--------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/docs/research/node-local-media-design.md b/docs/research/node-local-media-design.md index 1ccecb116..79aa47218 100644 --- a/docs/research/node-local-media-design.md +++ b/docs/research/node-local-media-design.md @@ -492,33 +492,21 @@ selectable only through real codec output and the normal library allocation. Configured ceilings are conservative and do not replace bandwidth probing; no cache of historical layer rates or additional polling owner is introduced. -### Capture Contract For The Next Implementation - -Use the existing LiveKit 2.22.1 screen-share construction as the initial output -set: original plus half-size at the same FPS, quarter bitrate with its 150 kbps -floor. Align derived NV12 dimensions down to even values. The group supports up -to three outputs, but this step does not invent a third product preset. - -The private capture command/probe contract advances to v5. Video arguments retain -the source profile and append one `--output WIDTH HEIGHT FPS BITRATE` per output, -ordered low to high. All current producers/readers change together, without v4 -fallback. Output profiles are codec bounds, not user preset choices. - -SMED v2 has a 32-byte big-endian header: magic[0:4], version[4], kind[5], flags[6], -layer[7], timestamp100ns[8:16], duration100ns[16:24], width[24:26], height[26:28], -payload length[28:32]. Video carries its output dimensions. PCM/status dimensions -and layer are zero. Kind 5 is a zero-payload input-frame begin with dimensions/ -layer/flags zero and a valid source timestamp/duration; it precedes all outputs -for that input even when the active prefix is zero. Kind 6 reports an unavailable -output layer with a bounded UTF-8 diagnostic, zero timing/dimensions/flags. - -Both pipe directions use this envelope. Kind 7 carries one bounded ASCII control -command (at most 64 bytes), with all other header fields zero: `Q`, `K layer` -(`-1` means all outputs), `A activeCount`, `B layer bitrate`. Reuse the existing -command grammar instead of mixing a line reader with binary video on stdin. Keyframe -requests consumed at encode start are restored on failure; requests arriving -during encoding must not be erased when an older keyframe completes. Control -wakeup is I/O work, not another media-quality scheduler. +### Capture Boundary + +Output construction follows the [media-quality owner](../standards/media-quality.md): +original plus a half-size output at the same FPS, with lower bitrate bounded by +the source ceiling. Derived NV12 dimensions round down to even values. The +group's bounded output slots do not add product presets. + +The current [capture reader](../../internal/app/nativecapture/protocol.go) owns +frame layout, kind-specific validation and allocation limits; [status](../status.md) +owns the matched bundle's protocol versions. Both pipe directions use that framed +boundary. Input-frame events precede per-output results, including when no output +is active; a lower-output failure remains distinct from loss of the source. +Keyframe requests consumed at encode start are restored on failure; requests +arriving during encoding must survive an older keyframe completing. Control wakeup +is I/O work, not another media-quality scheduler. Every slot owns its encoder/converter. Windows must retain one safe copy of the WGC input surface shared by slot workers, not reuse a returned frame-pool texture. From 4f508d3f5f820a1c5a9a0c15cc9ab9229da8767d Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:23:00 +0800 Subject: [PATCH 158/262] docs(guide): align custom quality instructions with candidate limits --- docs/guide/getting-started.md | 10 +++++++--- docs/guide/getting-started.zh-CN.md | 6 ++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 9f8d3d559..f6d2e6265 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -167,10 +167,14 @@ sharing to allow that fallback. The project's online site uses P2P only. Open **Sharing settings** below the picture to adjust picture and audio quality. - Choose a picture preset, or enter width and height under **Resolution ceiling**. - Piik preserves the source's proportions within that ceiling. Selecting 4K sets - the frame-rate ceiling to at most 30 fps. + Piik preserves the source's proportions within that ceiling. 4K supports a + frame-rate ceiling of up to 60 fps. +- Use the sliders or enter a **Frame rate ceiling** from 1 to 60 fps and a + **Video bitrate ceiling** from 0.1 to 50 Mbps. - Choose an audio preset, or enter an **Audio bitrate limit** from 6 to 320 kbps. - Picture and sound still adapt to the source, device and available bandwidth. + +The controls reflect the current device's supported limits. Picture and sound +still adapt to the source, device and available bandwidth. Piik remembers your applied settings for the next visit. Browser-only use remembers them for the current site; opening Piik through an updated App also keeps them diff --git a/docs/guide/getting-started.zh-CN.md b/docs/guide/getting-started.zh-CN.md index 82b0d44cc..b26064059 100644 --- a/docs/guide/getting-started.zh-CN.md +++ b/docs/guide/getting-started.zh-CN.md @@ -147,9 +147,11 @@ Linux 原生采集还需要[系统组件](../../native/capture/linux/README.md) 在画面下方打开 **分享设置**,调整画质和音频质量。 - 可选画质预设,也可在 **分辨率上限** 中填写宽和高。画面在上限内保留来源比例; - 选择 4K 时,帧率上限最高为 30 帧。 + 4K 可搭配最高 60 帧的帧率上限。 +- 可拖动滑块,也可直接填写 1–60 帧的 **帧率上限** 和 0.1–50 Mbps 的 **视频码率上限**。 - 音频保留三档预设,也可填写 6–320 kbps 的 **音频码率上限**。 - 实际画质和声音仍会根据来源、设备与可用带宽调整。 + +控件会按当前设备能力限制可选范围。实际画质和声音仍会根据来源、设备与可用带宽调整。 Piik 会记住已应用的设置,供下次使用。仅用浏览器时,设置保存在当前站点; 通过新版 App 打开时,更换公网邀请网址也能保留设置。 From 462c7d798a834bd9e8a82f258a23c81a74ff3aec Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:42:33 +0800 Subject: [PATCH 159/262] test(media): verify live rate limits across native and SFU routes --- docs/research/native-client-media.md | 19 +++++++++++++++- docs/todo.md | 9 ++++---- scripts/app-native-host-gate.ts | 34 +++++++++++++++++++++++++--- scripts/embedded-sfu-gate.ts | 13 +++++++++++ scripts/embedded-sfu-page.ts | 13 ++++++++--- 5 files changed, 76 insertions(+), 12 deletions(-) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index d14b39ec6..9bb42a0ff 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -680,7 +680,7 @@ These local checks do not establish 4K60, physical 4K/game capture, sustained device performance, all Browser decoders or all route changes. The existing SFU gate accepts `PIIK_EMBEDDED_SFU_VIDEO_SIZE=WIDTHxHEIGHT` for a -bounded 30 fps source; its default 720p15 control remains unchanged. Run +bounded 60 fps / 50 Mbps source; its default 720p15 control remains unchanged. Run `npm run gate:embedded-sfu` with `PIIK_EMBEDDED_SFU_GATE=true`, `PIIK_EMBEDDED_SFU_CODEC=h264` and `CHROME_PATH`; add `PIIK_EMBEDDED_SFU_DISPLAY=true` for actual tab capture. Synthetic 2160x3840 and @@ -771,6 +771,23 @@ bounded check does not cover room routing or physical-network recovery. The 1 fp preflight passed; the 4K60 preflight selected VP8 on this machine even though explicit H.264 delivered frames. Admission is not a sustained-cadence guarantee. +The Windows Native Host gate now exercises the actual numeric UI at 1 fps / +100 kbps and 60 fps / 50 Mbps with both codecs. Fresh decoded frames retain the +same Viewer stream through those changes, 4K/custom ceilings, pause, source +replacement and Viewer recovery. Its generated window is smaller than 4K; +this establishes ceiling application and continuity, not physical 4K capture. +The Browser H.264 and VP8 SFU controls use actual 3840x2160 synthetic input at +60 fps, decode high and lower outputs, retain the connection across both rate +bounds and audio updates, and recover subscriptions after reload. Positive +output does not establish sustained 60 fps on arbitrary devices or networks. + +The current Linux candidate also passes C11 compilation with warnings as errors +and the headless GStreamer regressions, including low/high rate admission, +source geometry, retirement and distinct Portal node-ID/serial targeting. +This uses GStreamer 1.28.3 and libportal 0.9.1 on an isolated Alpine runner; +its probe correctly reports no capture without a desktop session. Actual +Portal capture and platform packaging remain separate checks. + A uniform-noise 4K1/50 Mbps stress input reached the retained 4 MiB access-unit guard in both Native codecs; VP8 produced a 6,199,837-byte frame. The output was rejected rather than expanding allocation limits. Large bitrate/low-cadence diff --git a/docs/todo.md b/docs/todo.md index 2aab2404d..b7f401235 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -19,9 +19,9 @@ Git/PRs own completed history. - [ ] **Linux Portal target identity acceptance (#470).** The [contributor fix](https://github.com/TNTcraftHIM/Piik/pull/470) is integrated with authorship retained: serial and node-ID targeting remain distinct. - Run the candidate's Linux native checks and verify a Portal that omits - `pipewire-serial`. Upstream source review confirms the property distinction; - contributor device evidence does not replace candidate acceptance. + Candidate compilation and headless targeting checks pass. Verify an actual + Portal that omits `pipewire-serial`; the headless check does not exercise + desktop capture. - [ ] **Reconcile open PRs with the final candidate.** [#462](https://github.com/TNTcraftHIM/Piik/pull/462)'s 256/320 kbps request and [#469](https://github.com/TNTcraftHIM/Piik/pull/469)'s 4K request overlap the @@ -77,8 +77,7 @@ Implementation needs a concrete useful result within the product model. The inherited menu-only source range is replaced by 1..60 fps / 0.1..50 Mbps, with coherent lower-output bounds and actual-progress preflight. Local production-codec, source-change and Browser loopback checks pass; full-route - acceptance must include the low-rate settings as well as 4K60. The Linux native - build and range tests remain pending. + acceptance must include the low-rate settings as well as 4K60. Keep the audio sender ceiling backed by the existing packet-size evidence. Numeric audio and video controls are implemented; the assessment owns local capture, codec, derived-output, SFU and bounded mixed-route evidence. diff --git a/scripts/app-native-host-gate.ts b/scripts/app-native-host-gate.ts index 11abe0873..7c64376e8 100644 --- a/scripts/app-native-host-gate.ts +++ b/scripts/app-native-host-gate.ts @@ -84,6 +84,7 @@ interface GateResult { liveQualityChanged: boolean; fourKQualityChanged: boolean; customQualityChanged: boolean; + rateQualityChanges: number; pausedQualityChanged: boolean; backgroundProfileRecovery: boolean | null; sourceWindowRecovery: boolean | null; @@ -636,6 +637,7 @@ async function main(): Promise { liveQualityChanged: false, fourKQualityChanged: false, customQualityChanged: false, + rateQualityChanges: 0, pausedQualityChanged: false, backgroundProfileRecovery: mode === "local" && sourceKind === "window" ? false : null, sourceWindowRecovery: mode === "local" && sourceKind === "window" ? false : null, @@ -986,7 +988,7 @@ async function main(): Promise { `(() => { const resolution = document.querySelector('button.lr-chip[aria-label="1440p"]'); const preference = document.querySelector('button[name="degradationPreference"][value="maintain-framerate"]'); - const sliders = [...document.querySelectorAll('.lr-slider input[type="range"]')]; + const sliders = [...document.querySelectorAll('.lr-quality-rate input[type="range"]')]; return Boolean( resolution && preference && !resolution.disabled && !preference.disabled && sliders.length === 2 && sliders.every((slider) => !slider.disabled) @@ -1025,7 +1027,7 @@ async function main(): Promise { stage = "native-4k-preset"; await clickHostControl(cdp, host, `document.querySelectorAll('button.lr-tile')[2]`); await waitForValue(deadline => evaluate(cdp!, host, - `document.querySelector('.lr-slider input[type="range"]')?.value === '60' && + `document.querySelector('input[name="maxFramerate"]')?.value === '60' && document.querySelector('.lr-sharing-panel')?.getAttribute('aria-busy') === 'false'`, deadline), Boolean, 20_000); stage = "native-4k-controls"; @@ -1034,7 +1036,7 @@ async function main(): Promise { await evaluate(cdp, host, `document.querySelector('button.lr-chip[aria-label="4K"]')?.click()`, Date.now() + 5_000); await waitForValue(deadline => evaluate(cdp!, host, `(() => { - const fps = document.querySelector('.lr-slider input[type="range"]'); + const fps = document.querySelector('input[name="maxFramerate"]'); const width = document.querySelector('.lr-video-size input[name="width"]'); const height = document.querySelector('.lr-video-size input[name="height"]'); return fps?.max === '60' && fps.value === '60' && width?.value === '3840' && height?.value === '2160'; @@ -1064,6 +1066,31 @@ async function main(): Promise { video.getVideoPlaybackQuality().totalVideoFrames >= ${framesBeforeCustom + 10}); })()`, deadline), Boolean, 20_000); await assertVideoCodec(cdp, viewer, actualCodec); + stage = "native-rate-range"; + for (const [fps, mbps] of [[1, 0.1], [60, 50]]) { + for (const [name, value] of [["maxFramerate", fps], ["maxBitrate", mbps]] as const) { + await evaluate(cdp, host, `(() => { + const input = document.querySelector('input[name="${name}"]'); + if (!input || input.disabled) throw new Error('Rate control is unavailable'); + input.focus(); input.value = '${value}'; + if (!input.validity.valid) throw new Error('Rate control rejected the candidate range'); + input.blur(); + })()`, Date.now() + 5_000); + await waitForValue(deadline => evaluate(cdp!, host, + `document.querySelector('input[name="${name}"]')?.value === '${value}' && + document.querySelector('.lr-sharing-panel')?.getAttribute('aria-busy') === 'false'`, + deadline), Boolean, 20_000); + } + const before = await evaluate(cdp, viewer, + `document.querySelector('video')?.getVideoPlaybackQuality().totalVideoFrames ?? 0`, Date.now() + 5_000); + await waitForValue(deadline => evaluate(cdp!, viewer, `(() => { + const video = document.querySelector('video'); + return Boolean(video && video.srcObject === window.__piikGateMedia && + video.getVideoPlaybackQuality().totalVideoFrames >= ${before + 3}); + })()`, deadline), Boolean, 20_000); + await assertVideoCodec(cdp, viewer, actualCodec); + result.rateQualityChanges += 1; + } const framesBeforePreference = await evaluate( cdp, viewer, @@ -1521,6 +1548,7 @@ async function main(): Promise { result.viewerHeight > 0 && result.viewerHeight <= 1440 && result.codecPreserved : result.viewerConnected && result.viewerFrames >= 30 && result.qualityControlsEnabled && result.liveQualityChanged && result.fourKQualityChanged && result.customQualityChanged && + result.rateQualityChanges === 2 && result.backgroundProfileRecovery !== false && result.sourceWindowRecovery !== false && result.livePresetChanges === 2 && diff --git a/scripts/embedded-sfu-gate.ts b/scripts/embedded-sfu-gate.ts index 8e5c4feca..b1cb668b8 100644 --- a/scripts/embedded-sfu-gate.ts +++ b/scripts/embedded-sfu-gate.ts @@ -104,6 +104,7 @@ async function main(): Promise { low: null as { frames: number; width: number; height: number } | null, audioEnergy: { high: 0, low: 0, recovered: 0 }, audioUpdates: [] as Array<{ bitrate: number; frames: number; connectionRetained: boolean; energy: number }>, + videoRateUpdates: [] as Array<{ fps: number; bitrate: number; frames: number; connectionRetained: boolean }>, audioDiagnostics: null as unknown, secondRoom: twoRooms ? { connected: false, framesAfterFirstStopped: 0, audioBefore: 0, audioAfter: 0, stopped: false } : null, host: null as Snapshot | null, viewer: null as Snapshot | null, @@ -297,6 +298,17 @@ async function main(): Promise { if (!retained || energy <= previousEnergy) throw new Error("Audio update interrupted the current media connection"); previousEnergy = energy; } + result.stage = "numeric-video-rate-update"; + const videoRates = [[1, 100_000], requestedSize ? [60, 50_000_000] : [15, 2_000_000]]; + for (const [fps, bitrate] of videoRates) { + if (!await call(host, `gate.setVideoRates(${fps}, ${bitrate})`)) throw new Error("Numeric video rate update failed"); + const frames = await call<{ frames: number }>(viewer, + `gate.waitForFrames(${result.high!.width}, ${result.high!.height}, 3)`); + const after = await call(viewer, "gate.snapshot()"); + const retained = after.connectionId === audioConnection && after.committed; + result.videoRateUpdates.push({ fps, bitrate, frames: frames.frames, connectionRetained: retained }); + if (!retained) throw new Error("Video rate update replaced the current media connection"); + } if (displayArm) { result.stage = "display-background"; if (result.host!.capture?.displaySurface !== "browser" || result.host!.visibility !== "hidden") { @@ -413,6 +425,7 @@ async function main(): Promise { (nativeArm || result.stereoSeparationDb?.every(db => db > 20) === true) && result.viewer.audioKbps > 0 && result.audioEnergy.high > 0 && result.audioEnergy.low > result.audioEnergy.high && result.high !== null && result.low !== null && result.audioEnergy.recovered > 0 && + result.videoRateUpdates.length === 2 && result.videoRateUpdates.every(update => update.connectionRetained) && result.publicationRetired && result.subscriptionRecovered && result.pageReloadRecovered && result.cleanup && result.nativeShareStopped && (!displayArm || result.displayLifecycle?.hiddenFrames === 150 && result.displayLifecycle.lowLayerDecoded && result.displayLifecycle.resumed && result.displayLifecycle.replaced) && diff --git a/scripts/embedded-sfu-page.ts b/scripts/embedded-sfu-page.ts index 577878715..123cdeb56 100644 --- a/scripts/embedded-sfu-page.ts +++ b/scripts/embedded-sfu-page.ts @@ -199,7 +199,7 @@ async function connect(room: CreateRoomResponse, host: boolean): Promise { export async function startHost(nativeSource?: { title: string; port: number }, selectedCodec: "vp8" | "h264" = "vp8", display = false, size?: QualitySettings["resolution"]): Promise { codec = selectedCodec; - profile = size ? { ...high, resolution: size, maxFramerate: 30, maxBitrate: 8_000_000 } : high; + profile = size ? { ...high, resolution: size, maxFramerate: 60, maxBitrate: 50_000_000 } : high; const response = await fetch("/api/rooms", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ codeEntryPolicy: "private" }), @@ -342,8 +342,7 @@ export async function audioEnergy(): Promise { return await probe(); } -export async function setAudioBitrate(maxAudioBitrate: number): Promise { - const next = { ...profile, maxAudioBitrate }; +async function applyProfile(next: QualitySettings): Promise { if (native) await native.updateShare(shareGeneration, next); if (!await hostRoute!.updateProfile(next)) return false; profile = next; @@ -351,6 +350,14 @@ export async function setAudioBitrate(maxAudioBitrate: number): Promise return true; } +export async function setAudioBitrate(maxAudioBitrate: number): Promise { + return await applyProfile({ ...profile, maxAudioBitrate }); +} + +export async function setVideoRates(maxFramerate: number, maxBitrate: number): Promise { + return await applyProfile({ ...profile, maxFramerate, maxBitrate }); +} + export async function waitForFrames(width: number, height: number, minimumFrames = 15, withinCeiling = false, tolerance = 0) { if (!video) throw new Error("Viewer video is unavailable"); const target = video; From f7556923fadb9b5f6ba46a9b0e8a7f57071622fd Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:46:50 +0800 Subject: [PATCH 160/262] fix(app): retain STUN discovery redundancy without NAT prediction --- docs/adr/0009-optional-nat-prediction.md | 12 +++--- .../adr/0010-cross-platform-client-runtime.md | 5 ++- docs/guide/troubleshooting.md | 6 +++ docs/guide/troubleshooting.zh-CN.md | 5 +++ docs/research/nat-traversal.md | 37 +++++++++++++----- docs/todo.md | 2 + internal/app/app.go | 14 ++++--- internal/app/app_test.go | 4 +- .../app/mediaedge/candidate_gathering_test.go | 39 +++++++++++++++++++ tests/nat-prediction.test.ts | 10 +++++ 10 files changed, 110 insertions(+), 24 deletions(-) diff --git a/docs/adr/0009-optional-nat-prediction.md b/docs/adr/0009-optional-nat-prediction.md index 28d9ad34a..81b61f75a 100644 --- a/docs/adr/0009-optional-nat-prediction.md +++ b/docs/adr/0009-optional-nat-prediction.md @@ -25,9 +25,10 @@ connection acquisition reuses the route controller's existing opportunity ledger switch defaults on, remains Host-controllable, locks while sharing, and is not persisted beyond that share generation. 2. An enabled Site derives ports 3479 and 3480 from its first ordinary STUN - authority on UDP 3478. Public-link App mode instead supplies one ordinary - public STUN destination and two fixed public survey destinations; pure LAN - mode supplies none. The same share switch and ICE configuration cover every + authority on UDP 3478. Public-link App mode supplies three ordinary public + STUN destinations, reusing the same set for its prediction survey; disabling + prediction retains ordinary discovery redundancy. Pure LAN mode supplies + none. The same share switch and ICE configuration cover every Browser or Native P2P edge. SFU PeerConnections remain unchanged. 3. For one ICE gathering generation, the adapter observes only UDP `srflx` candidates belonging to the exact three-destination survey set. Browser @@ -84,8 +85,9 @@ Negative: - the Browser may still reject synthetic candidates or the NAT may not have a matching mapping; -- enabled P2P connections perform two extra STUN transactions and - may add connectivity checks for up to eight adjacent UDP ports; +- Hosted prediction adds two survey destinations; public-link Apps reuse their + ordinary destinations. Prediction may add connectivity checks for up to eight + adjacent UDP ports; - two endpoint-dependent NATs remain sensitive to candidate scheduling and intervening mappings, so room-wide coverage is not a success guarantee; and - enabled acquisition can occupy up to four ordinary operation windows when diff --git a/docs/adr/0010-cross-platform-client-runtime.md b/docs/adr/0010-cross-platform-client-runtime.md index 1f113fcc8..0a5b17974 100644 --- a/docs/adr/0010-cross-platform-client-runtime.md +++ b/docs/adr/0010-cross-platform-client-runtime.md @@ -141,8 +141,9 @@ topologies and makes a saved Site unavailable while another room source runs. 10. Local mode is one explicit server composition: static current assets, memory-only rooms, peer-assisted media, no SQLite or SFU, and localhost plus current LAN IPv4 origins. It uses no public discovery by default. Public-link - mode adds its temporary HTTPS origin, one ordinary public STUN destination, - and two bounded public survey destinations. Site mode consumes that Site's + mode adds its temporary HTTPS origin and the bounded public STUN discovery + and survey configuration in [ADR-0009](0009-optional-nat-prediction.md). + Site mode consumes that Site's configured STUN survey. Both feed the same connection-local prediction adapter; Native additionally owns its UDP socket and best-effort port mapping. 11. One App configuration owns the optional Site origin and an optional, diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 350551cfe..3ee5e3f02 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -259,6 +259,12 @@ Routers and providers differ in how they map addresses and admit incoming traffi some combinations make direct connections difficult. Firewalls and browser policies can also block media. The message alone does not identify the cause. +STUN helps discover an address for direct connections; it does not relay the +picture. An unreachable STUN service can therefore leave chat working while +media fails. On self-hosted sites, administrators should check that `STUN_URLS` +points to reachable UDP services; an HTTP reverse proxy does not forward STUN. +Compare the diagnostic reports from both peers before assigning the cause. + Try these in order, stopping when the picture arrives: 1. **Retry the viewing page.** Choose **Reconnect** if it is available. If it is diff --git a/docs/guide/troubleshooting.zh-CN.md b/docs/guide/troubleshooting.zh-CN.md index 7b499f6ea..fa2db54f3 100644 --- a/docs/guide/troubleshooting.zh-CN.md +++ b/docs/guide/troubleshooting.zh-CN.md @@ -217,6 +217,11 @@ NAT(网络地址转换)让多台设备共用一个上网地址。不同路 外部数据进入的规则不同,有些组合会让设备难以直连。防火墙和浏览器策略也可能阻止媒体连接, 仅凭这条提示还不能确定具体原因。 +STUN 帮助设备发现用于直连的地址,不负责转发画面。STUN 服务不可达时, +也可能出现聊天正常、媒体连不上的情况。自部署管理员应检查 `STUN_URLS` +是否指向可达的 UDP 服务;HTTP 反向代理不能转发 STUN。 +请结合双方同一次尝试的诊断报告判断原因。 + 可以依次尝试,画面恢复后就不必继续: 1. **重新尝试观看连接。** 如果 **重新连接** 可用,先点击;按钮不可用或没有效果时, diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index b6d53fc49..76f64d622 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -1,6 +1,6 @@ # Browser And Native NAT Traversal -Last reviewed: 2026-10-10 +Last reviewed: 2026-10-11 This document owns evidence for improving direct ICE without adding a new relay or a custom transport. The current product contract remains standard @@ -15,10 +15,9 @@ WebRTC ICE with STUN discovery and bounded SFU fallback. - A STUN operator observes the client's source address, source port, request time and protocol metadata. It never receives the DTLS-SRTP media path. An external STUN endpoint is therefore a metadata dependency, not a media relay. -- The current `STUN_URLS` wire already accepts a bounded list. Adding an - auxiliary STUN destination needs no client compatibility surface, but it has - no product value until a valid Browser observation or prediction consumer - exists. +- The current `STUN_URLS` wire already accepts a bounded list. Independent + ordinary endpoints can preserve discovery when one destination is unreachable; + additional observations alone do not establish better hole punching. On 2026-08-31, Chrome 151 on Windows gathered a host plus IPv4 srflx candidate through the self-hosted STUN endpoint and independently through Cloudflare's @@ -27,6 +26,25 @@ configured, Chromium emitted one deduplicated srflx candidate. No raw address or port was retained. This proves endpoint reachability and candidate deduplication; it does not classify the NAT or prove another direct path. +### Public-Link Discovery Redundancy + +A v1.9.2 report describes working chat but failed media, with only MiWiFi STUN +responding on the reporter's network. It supplies neither paired ICE traces nor +the share's prediction setting. The code confirms a narrower configuration gap: +disabling prediction also removed both fallback discovery destinations, leaving +Cloudflare alone. The candidate keeps all three existing destinations in ordinary +ICE and reuses that set for prediction. It adds no provider, polling, URL ranking, +fallback transport or new configuration surface; Local LAN remains unchanged. + +On 2026-10-11, local Chromium gathered a server-reflexive candidate from each +destination independently and from the combined list. Some `701` error events +coexisted with valid candidates, so an error event alone cannot establish complete +discovery failure. Native regression checks confirm a healthy destination returns +without waiting for silent UDP or unresolved DNS, and cancellation retires the +survey. These controls do not reproduce the reporter's network or establish its +final connection cause. The [WebRTC event specification](https://www.w3.org/TR/webrtc/#rtcicecandidateerrorevent) +owns error semantics; compare the attempt's gathered and selected candidates too. + ### SFU Behind Server NAT An isolated Linux kernel-NAT control on 2026-10-09 places the current Server @@ -88,10 +106,11 @@ supports an opt-in mechanism, not a default or a participant-wide classifier. ## Current Decision -An ordinary STUN endpoint remains the discovery service. A Site may enable -same-host STUN-only listeners on 3479 and 3480. Public-link App mode supplies -one ordinary public endpoint plus two public survey destinations; pure LAN mode -supplies none. Available authorities expose the same per-share Host switch, +Ordinary STUN remains the discovery service. A Site may enable same-host +STUN-only listeners on 3479 and 3480. Public-link App mode supplies three ordinary +public destinations and reuses them for prediction, so disabling prediction +does not remove discovery redundancy; pure LAN mode supplies none. +Available authorities expose the same per-share Host switch, which defaults on and remains locked while sharing. The accepted adapter is deliberately connection-local and additive. It waits for a diff --git a/docs/todo.md b/docs/todo.md index b7f401235..3180f055b 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -155,6 +155,8 @@ of a reporter's cause; research owns the completed experiments and their limits. - [ ] **No available media route.** Compare the same endpoints/networks before assigning a regression since v1.4. Obtain paired Host/Viewer/App reports and distinguish route allocation, ICE, first-frame admission and route rejection. + Retest the public-invitation STUN report with ordinary discovery redundancy; + its endpoint timeout results lack paired ICE traces and the prediction setting. [#443](https://github.com/TNTcraftHIM/Piik/issues/443) lacks a connected Native edge and its paired Viewer/App transport report; [#448](https://github.com/TNTcraftHIM/Piik/issues/448) only reaches route allocation. diff --git a/internal/app/app.go b/internal/app/app.go index e112ad100..a9bd6859a 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -33,10 +33,10 @@ import ( ) const ( - DefaultLocalPort = 8787 - publicSTUNURL = "stun:stun.cloudflare.com:3478" - publicNATPredictionSTUNURLA = "stun:stun.miwifi.com:3478" - publicNATPredictionSTUNURLB = "stun:stun.chat.bilibili.com:3478" + DefaultLocalPort = 8787 + publicSTUNURL = "stun:stun.cloudflare.com:3478" + publicFallbackSTUNURLA = "stun:stun.miwifi.com:3478" + publicFallbackSTUNURLB = "stun:stun.chat.bilibili.com:3478" // localShutdownTimeout bounds the ordered Local stop. The packaged smoke // allows 10 s between the stop request and the exit, so it stays well // inside that budget. @@ -653,12 +653,14 @@ func discoverNativeMedia(ctx context.Context, configuredPath string) nativeRunti // has a public path, so only --link configures public STUN and the bounded NAT // prediction survey. Native edges receive ICE configuration from that authority; // serverconfig.Local deliberately does not inherit Hosted environment settings. +// All three destinations also serve ordinary discovery: disabling prediction +// must not make public-link ICE depend on a single STUN operator. func localSTUNURLs(link bool) (stunURLs []string, natPredictionSTUNURLs []string) { if !link { return nil, nil } - return []string{publicSTUNURL}, - []string{publicNATPredictionSTUNURLA, publicNATPredictionSTUNURLB} + return []string{publicSTUNURL, publicFallbackSTUNURLA, publicFallbackSTUNURLB}, + []string{publicFallbackSTUNURLA, publicFallbackSTUNURLB} } // endLocalServer is step 1 of the ADR-0010 item 12 unwind: every room ends, diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 885abf1a4..88f90a8ec 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -712,9 +712,9 @@ func TestLocalServerOwnsItsSTUNConfiguration(t *testing.T) { {name: "local"}, { name: "link", link: true, - stunURLs: []string{publicSTUNURL}, + stunURLs: []string{publicSTUNURL, publicFallbackSTUNURLA, publicFallbackSTUNURLB}, natPredictionSTUNURLs: []string{ - publicNATPredictionSTUNURLA, publicNATPredictionSTUNURLB, + publicFallbackSTUNURLA, publicFallbackSTUNURLB, }, }, } { diff --git a/internal/app/mediaedge/candidate_gathering_test.go b/internal/app/mediaedge/candidate_gathering_test.go index 694fd2a0d..2c8a132a8 100644 --- a/internal/app/mediaedge/candidate_gathering_test.go +++ b/internal/app/mediaedge/candidate_gathering_test.go @@ -169,3 +169,42 @@ func TestSurveyReturnsWhenEveryDestinationAlreadyFailed(t *testing.T) { t.Fatalf("completed failure retained its survey timer: %v", elapsed) } } + +func TestSurveyHealthySTUNDoesNotWaitForSilentDestination(t *testing.T) { + engine, err := NewEngine(EngineOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = engine.Close() }) + socket := testICESocket(t, engine) + silent, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = silent.Close() }) + servers := []webrtc.ICEServer{{URLs: []string{ + "stun:" + silent.LocalAddr().String(), localSurveyServer(t), + }}} + observations := make(chan mappedAddress, 1) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan struct{}) + go func() { + defer close(done) + socket.surveySTUN(ctx, servers, func(mapped mappedAddress) { observations <- mapped }) + }() + select { + case mapped := <-observations: + if mapped.port != socket.localPort { + t.Fatal("fallback STUN did not observe the media socket") + } + case <-time.After(time.Second): + t.Fatal("healthy fallback waited for the silent STUN destination") + } + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("retired survey kept waiting for the silent destination") + } +} diff --git a/tests/nat-prediction.test.ts b/tests/nat-prediction.test.ts index 84db6e3ac..bd9587c41 100644 --- a/tests/nat-prediction.test.ts +++ b/tests/nat-prediction.test.ts @@ -149,6 +149,16 @@ describe("NAT prediction ICE adapter", () => { expect(natPredictionSurveyUrls(servers)).toEqual(new Set()); }); + it("retains ordinary fallback discovery independently of prediction", () => { + const urls = ["stun:primary.test:3478", "stun:fallback-a.test:3478", "stun:fallback-b.test:3478"]; + const servers = [{ urls }]; + const auxiliary = urls.slice(1); + for (const enabled of [false, true]) { + expect(iceServersWithNatPrediction(servers, enabled, auxiliary)).toEqual(servers); + } + expect(natPredictionSurveyUrls(servers, auxiliary)).toEqual(new Set(urls)); + }); + it.each([ ["stun:Share.TEST", "stun:share.test:3478"], ["STUN:Share.TEST:03478", "stun:share.test:3478"], From 09fd1c29f1cbca3c3f745371b36bb5eef889fe4c Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:59:04 +0800 Subject: [PATCH 161/262] test(media): isolate candidate diagnostic logging --- internal/app/mediaedge/edge.go | 6 +++--- internal/app/mediaedge/edge_debug_test.go | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/internal/app/mediaedge/edge.go b/internal/app/mediaedge/edge.go index 9caf9617d..38bfa51b9 100644 --- a/internal/app/mediaedge/edge.go +++ b/internal/app/mediaedge/edge.go @@ -432,7 +432,7 @@ func (edge *Edge) Close() error { edge.pendingCandidates = nil edge.mu.Unlock() if edge.local && slog.Default().Enabled(edge.engine.ctx, slog.LevelDebug) { - logLocalMediaCandidateCounts(edge.connection.GetStats()) + logLocalMediaCandidateCounts(slog.Default(), edge.connection.GetStats()) } err := edge.transport.Close() edge.socket.close() @@ -444,7 +444,7 @@ func (edge *Edge) Close() error { return err } -func logLocalMediaCandidateCounts(report webrtc.StatsReport) { +func logLocalMediaCandidateCounts(logger *slog.Logger, report webrtc.StatsReport) { for _, direction := range []webrtc.StatsType{webrtc.StatsTypeLocalCandidate, webrtc.StatsTypeRemoteCandidate} { for _, kind := range []webrtc.ICECandidateType{webrtc.ICECandidateTypeHost, webrtc.ICECandidateTypeSrflx, webrtc.ICECandidateTypePrflx, webrtc.ICECandidateTypeRelay} { udp, tcp := 0, 0 @@ -460,7 +460,7 @@ func logLocalMediaCandidateCounts(report webrtc.StatsReport) { tcp++ } } - slog.Debug("piik-client", "event", "media-candidate-counts", "local", true, + logger.Debug("piik-client", "event", "media-candidate-counts", "local", true, "direction", string(direction), "type", kind.String(), "udp", udp, "tcp", tcp) } } diff --git a/internal/app/mediaedge/edge_debug_test.go b/internal/app/mediaedge/edge_debug_test.go index 419fc5c14..c8930f283 100644 --- a/internal/app/mediaedge/edge_debug_test.go +++ b/internal/app/mediaedge/edge_debug_test.go @@ -12,10 +12,8 @@ import ( func TestLocalMediaCandidateCountsExcludePrivateFields(t *testing.T) { var output bytes.Buffer - previous := slog.Default() - slog.SetDefault(slog.New(slog.NewJSONHandler(&output, &slog.HandlerOptions{Level: slog.LevelDebug}))) - defer slog.SetDefault(previous) - logLocalMediaCandidateCounts(webrtc.StatsReport{ + logger := slog.New(slog.NewJSONHandler(&output, &slog.HandlerOptions{Level: slog.LevelDebug})) + logLocalMediaCandidateCounts(logger, webrtc.StatsReport{ "private-local": webrtc.ICECandidateStats{ ID: "private-local", Type: webrtc.StatsTypeLocalCandidate, CandidateType: webrtc.ICECandidateTypeHost, Protocol: "udp", From 1dbfa207c3bb4fb20de59692d7454d67a574ae00 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 17:15:41 +0800 Subject: [PATCH 162/262] test(capture): align macOS admission checks with input ownership Keep raw SPS and reference-storage validation at the shared Go input boundary. Check CoreMedia's visible format without assuming it preserves uncropped SPS dimensions, and retain both compiler output streams on packaging failures. --- docs/research/nat-traversal.md | 4 +++- docs/research/native-client-media.md | 20 +++++++++++++++----- docs/todo.md | 16 ++++++++-------- native/capture/darwin/README.md | 6 ++++-- native/capture/darwin/main.swift | 17 ++++++++++++----- scripts/assemble-app.mjs | 2 +- scripts/check-go.mjs | 2 +- scripts/package-app-candidate.mjs | 2 +- 8 files changed, 45 insertions(+), 24 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 76f64d622..23d2e6b4f 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -39,7 +39,9 @@ fallback transport or new configuration surface; Local LAN remains unchanged. On 2026-10-11, local Chromium gathered a server-reflexive candidate from each destination independently and from the combined list. Some `701` error events coexisted with valid candidates, so an error event alone cannot establish complete -discovery failure. Native regression checks confirm a healthy destination returns +discovery failure. The pinned Pion STUN client also received and decoded +`XOR-MAPPED-ADDRESS` from each destination; the fallback is not Browser-only. +Native regression checks confirm a healthy destination returns without waiting for silent UDP or unresolved DNS, and cancellation retires the survey. These controls do not reproduce the reporter's network or establish its final connection cause. The [WebRTC event specification](https://www.w3.org/TR/webrtc/#rtcicecandidateerrorevent) diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 9bb42a0ff..9b9fdefa1 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -1,6 +1,6 @@ # Native App Media Evidence -- Reviewed: 2026-10-06 +- Reviewed: 2026-10-11 - Scope: platform capture, shared encoded sources, Pion transport and Browser decode; current behavior belongs to [media quality](../standards/media-quality.md) - Status: Windows physical native Host and Viewer gates passed; current macOS @@ -776,10 +776,20 @@ The Windows Native Host gate now exercises the actual numeric UI at 1 fps / same Viewer stream through those changes, 4K/custom ceilings, pause, source replacement and Viewer recovery. Its generated window is smaller than 4K; this establishes ceiling application and continuity, not physical 4K capture. -The Browser H.264 and VP8 SFU controls use actual 3840x2160 synthetic input at -60 fps, decode high and lower outputs, retain the connection across both rate -bounds and audio updates, and recover subscriptions after reload. Positive -output does not establish sustained 60 fps on arbitrary devices or networks. +The Browser H.264 and VP8 SFU controls request 60 fps from actual 3840x2160 +synthetic input, decode high and lower outputs, retain the connection across +both rate bounds and audio updates, and recover subscriptions after reload. +Native WGC-to-SFU controls also pass both codecs and rate changes with the same +connection; their source is smaller than 4K. Positive output does not establish +sustained 60 fps on arbitrary devices or networks. + +A 3840x2160/60 Browser-to-Native fanout control failed the existing 30 fps +performance threshold: its source produced about 32..33 fps and the two Viewers +about 27 fps. A separate functional continuation retained that negative result +and passed live quality changes, pause/resume, source-audio replacement and +App-exit recovery. That run measured about 41 and 37 fps at the Viewers, still +below the requested 60 fps. The controls establish continuity on this machine, +not sustained 4K60 cadence; they do not justify custom quality floors or resets. The current Linux candidate also passes C11 compilation with warnings as errors and the headless GStreamer regressions, including low/high rate admission, diff --git a/docs/todo.md b/docs/todo.md index 3180f055b..3a0cc64a9 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -27,7 +27,7 @@ Git/PRs own completed history. [#469](https://github.com/TNTcraftHIM/Piik/pull/469)'s 4K request overlap the numeric-media candidate; neither PR has been merged. 4K60 is included in candidate admission, alongside the tested wider video control range, and still - needs route/device acceptance. Preserve + needs physical device/network acceptance. Preserve contributor credit and explain the disposition at integration. Rewrite [#459](https://github.com/TNTcraftHIM/Piik/pull/459)'s earlier maintenance scope and compatibility claims around the final 2.0 result before integration. @@ -76,18 +76,18 @@ Implementation needs a concrete useful result within the product model. resource budgets intact; opening an output range still needs end-to-end evidence. The inherited menu-only source range is replaced by 1..60 fps / 0.1..50 Mbps, with coherent lower-output bounds and actual-progress preflight. Local - production-codec, source-change and Browser loopback checks pass; full-route - acceptance must include the low-rate settings as well as 4K60. + production-codec, source-change, Browser loopback, Native Host and SFU checks + cover both rate bounds. Preserve these checks in final package acceptance. Keep the audio sender ceiling backed by the existing packet-size evidence. Numeric audio and video controls are implemented; the assessment owns local capture, codec, derived-output, SFU and bounded mixed-route evidence. Remaining receiver acceptance needs physical limited decoders and camera orientation behavior, beyond the bounded SDP-limit and mixed-route controls. - Browser display capture, Browser relay and Browser-to-Native fanout now have - bounded high-resolution evidence. macOS geometry SDK/hardware checks and - physical 4K/game capture remain open. Repeat live Native-to-Browser, derived - relay and SFU checks at 4K60; local codec and Browser loopback success does not - establish sustained full-route performance. Preserve safety limits and + Browser display capture, Browser relay, Browser-to-Native fanout and SFU have + bounded high-resolution continuity evidence. macOS geometry SDK/hardware + checks and physical 4K/game capture remain open. The fanout controls did not + establish sustained 60 fps; retain that performance limit when arranging + device/network acceptance. Preserve safety limits and framework-owned adaptation. The [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment) retain explicit receive-limit enforcement and the current 320 kbps sender diff --git a/native/capture/darwin/README.md b/native/capture/darwin/README.md index a2cf08487..a72a03dce 100644 --- a/native/capture/darwin/README.md +++ b/native/capture/darwin/README.md @@ -20,8 +20,10 @@ sh native/capture/darwin/build.sh /outside/repository/build ``` `--geometry-self-test` checks content cropping, Retina scaling and rejection of -invalid frame geometry, plus H.264 visible/coded-size admission, without requiring -a hardware encoder. `--self-test` +invalid frame geometry, plus H.264 format/aperture admission, without requiring +a hardware encoder. Go validates every raw SPS's coded size and reference budget +before sending encoded input to the helper; CoreMedia can normalize SPS cropping +in its format description. `--self-test` encodes successive landscape, portrait, ultrawide and small 420v buffers through one output worker, requiring the fitted dimensions and a constrained-baseline SPS/PPS/IDR after each size change. Neither needs Screen Recording permission. diff --git a/native/capture/darwin/main.swift b/native/capture/darwin/main.swift index 3bdc12036..8e7899a86 100644 --- a/native/capture/darwin/main.swift +++ b/native/capture/darwin/main.swift @@ -1184,10 +1184,15 @@ private func geometrySelfTest() throws { CGRect(x: 0, y: 0, width: 3840, height: 2160) else { throw CaptureFailure(description: "H.264 visible aperture self-test failed") } - let oversized = Data([0x67, 0x42, 0xc0, 0x33, 0xda, 0x00, 0xf0, 0x01, - 0xe1, 0xf8, 0x03, 0x49, 0x40]) + // Uncropped 4096x2160 exceeds the visible-picture budget. Raw SPS storage + // and reference counts are validated by Go before reaching this process; + // CoreMedia can fold SPS cropping into its reported format dimensions. + let oversized = Data([0x67, 0x42, 0xc0, 0x33, 0xdc, 0x01, 0x00, 0x01, + 0x0f, 0xb0, 0x16, 0xa0, 0x20, 0x20, 0x28, 0x00, + 0x00, 0x03, 0x00, 0x08, 0x00, 0x00, 0x03, 0x00, + 0x14, 0x78, 0xc1, 0x9c]) guard (try? h264InputFormat(sps: oversized, pps: pps)) == nil else { - throw CaptureFailure(description: "H.264 crop concealed oversized storage") + throw CaptureFailure(description: "H.264 oversized visible picture was accepted") } } @@ -1660,14 +1665,16 @@ private func h264InputFormat(sps: Data, pps: Data) throws -> CMVideoFormatDescri } try require(status, "decode-h264-format") guard let format else { throw CaptureFailure(description: "missing input format") } + // Go owns raw SPS storage/reference admission. CoreMedia may report a + // crop-normalized format; check that description and its aperture here. let dimensions = CMVideoFormatDescriptionGetDimensions(format) let width = Int(dimensions.width), height = Int(dimensions.height) guard width >= 2, width <= maxCodedVideoDimension, width % 2 == 0, height >= 2, height <= maxCodedVideoDimension, height % 2 == 0, width * height <= maxCodedVideoPixels else { - throw CaptureFailure(description: "decoded input storage exceeds its bound") + throw CaptureFailure(description: "decoded input format dimensions exceed their bound") } - // Encoded dimensions include padding; the clean aperture owns visible size. + // The clean aperture owns visible size when the format retains padding. let area = CMVideoFormatDescriptionGetCleanAperture(format, originIsAtTopLeft: true) let bounds = CGRect(x: 0, y: 0, width: width, height: height) guard area.minX.isFinite, area.minY.isFinite, area.width.isFinite, area.height.isFinite, diff --git a/scripts/assemble-app.mjs b/scripts/assemble-app.mjs index d7f337cc3..c42cc2707 100644 --- a/scripts/assemble-app.mjs +++ b/scripts/assemble-app.mjs @@ -40,7 +40,7 @@ function run(command, args, cwd, environment = process.env) { }); if (result.error) throw result.error; if (result.status !== 0) { - fail(`${command} failed: ${(result.stderr || result.stdout).trim()}`); + fail(`${command} failed: ${[result.stdout, result.stderr].filter(Boolean).join("\n").trim()}`); } return result.stdout.trim(); } diff --git a/scripts/check-go.mjs b/scripts/check-go.mjs index b84d1b65b..efcccaf25 100644 --- a/scripts/check-go.mjs +++ b/scripts/check-go.mjs @@ -38,7 +38,7 @@ function run(command, args, options = {}) { if (result.error) throw result.error; if (result.status !== 0) { const detail = options.capture - ? (result.stderr || result.stdout).trim() + ? [result.stdout, result.stderr].filter(Boolean).join("\n").trim() : ""; throw new Error(`${basename(command)} failed${detail ? `: ${detail}` : ""}`); } diff --git a/scripts/package-app-candidate.mjs b/scripts/package-app-candidate.mjs index ac8affc01..605ad1d8e 100644 --- a/scripts/package-app-candidate.mjs +++ b/scripts/package-app-candidate.mjs @@ -36,7 +36,7 @@ function run(command, args, cwd) { }); if (result.error) throw result.error; if (result.status !== 0) { - fail(`${basename(command)} failed: ${(result.stderr || result.stdout).trim()}`); + fail(`${basename(command)} failed: ${[result.stdout, result.stderr].filter(Boolean).join("\n").trim()}`); } if (result.stderr) process.stderr.write(result.stderr); return result.stdout.trim(); From 8207606b7a98cdbb80d9b99e98c423e432fb5a18 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 17:31:14 +0800 Subject: [PATCH 163/262] fix(build): retain newer Windows compiler runtime when linking --- native/capture/windows/README.md | 5 ++++- native/capture/windows/build.ps1 | 14 +++++++++++++- native/capture/windows/get-webrtc.ps1 | 1 + 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/native/capture/windows/README.md b/native/capture/windows/README.md index 51f7dd64e..76aac48f0 100644 --- a/native/capture/windows/README.md +++ b/native/capture/windows/README.md @@ -83,7 +83,10 @@ the old fixed one-frame VBV override is removed so later bitrate increases are not trapped by the original buffer limit. The build caches the pinned WebRTC SDK and matching official MSVC linker/runtime -libraries under `build/encoder-pool`. The SDK supplies libvpx, replacing the +libraries under `build/encoder-pool`. A newer installed MSVC keeps its own linker +and runtime; older installations use that pinned minimum, following +[Microsoft's binary compatibility rule](https://learn.microsoft.com/en-us/cpp/porting/binary-compat-2015-2017). +The SDK supplies libvpx, replacing the separate source build. Only statically linked code and required notices enter the package, not the SDK/toolchain or another runtime service. diff --git a/native/capture/windows/build.ps1 b/native/capture/windows/build.ps1 index 2a39a4098..a79838539 100644 --- a/native/capture/windows/build.ps1 +++ b/native/capture/windows/build.ps1 @@ -36,7 +36,19 @@ $includeFlags = @('', 'third_party\abseil-cpp', 'third_party\boringssl\src\inclu } $compileFlags = '/nologo /c /std:c++20 /EHsc /GR /O2 /W4 /WX /MT /D_ITERATOR_DEBUG_LEVEL=0 /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 /DNTDDI_VERSION=0x0A00000A /external:W0 ' + ($includeFlags -join ' ') $systemLibraries = 'ole32.lib mmdevapi.lib runtimeobject.lib user32.lib gdi32.lib dwmapi.lib shell32.lib mfplat.lib mf.lib mfuuid.lib d3d11.lib d2d1.lib dxgi.lib dxguid.lib evr.lib oleaut32.lib windowsapp.lib winmm.lib ws2_32.lib strmiids.lib crypt32.lib dmoguids.lib iphlpapi.lib msdmo.lib secur32.lib wmcodecdspuuid.lib' -$linkCommand = '"{0}" /nologo /libpath:"{1}"' -f $webrtc.Linker,$webrtc.RuntimeLibraries +# The linker and CRT must be at least as new as every input's toolchain. +# Keep the pinned WebRTC minimum for older local installations, but do not +# override a newer compiler's runtime on current Visual Studio/CI images. +$toolsEnvironment = & cmd.exe /d /s /c ('call "{0}" -arch=x64 -host_arch=x64 >nul && set VCToolsInstallDir' -f $developerCommand) +if ($LASTEXITCODE -ne 0) { throw 'Could not resolve the Visual Studio C++ toolchain.' } +$installedTools = ($toolsEnvironment | Where-Object { $_.StartsWith('VCToolsInstallDir=') }) -replace '^VCToolsInstallDir=', '' +if ([string]::IsNullOrWhiteSpace($installedTools)) { throw 'Visual Studio did not select a C++ toolchain.' } +$installedVersion = [version](Split-Path -Leaf $installedTools.TrimEnd('\')) +$linkCommand = if ($installedVersion -ge $webrtc.ToolchainVersion) { + 'link.exe /nologo' +} else { + '"{0}" /nologo /libpath:"{1}"' -f $webrtc.Linker,$webrtc.RuntimeLibraries +} function Invoke-CaptureBuild([string]$command) { & cmd.exe /d /s /c ('call "{0}" -arch=x64 -host_arch=x64 >nul && {1}' -f $developerCommand,$command) | Out-Host diff --git a/native/capture/windows/get-webrtc.ps1 b/native/capture/windows/get-webrtc.ps1 index b8958a4c5..b3de932a1 100644 --- a/native/capture/windows/get-webrtc.ps1 +++ b/native/capture/windows/get-webrtc.ps1 @@ -56,6 +56,7 @@ if ($sdkNoticeHash -ne $notice.sha256 -or } $msvcPath = Join-Path $toolchainPath $pins.msvc.directory return @{ + ToolchainVersion = [version](Split-Path -Leaf $msvcPath) Include = Join-Path $sdkPath 'include' Library = Join-Path $sdkPath 'lib\webrtc.lib' RuntimeLibraries = Join-Path $msvcPath 'lib\x64' From 2b8d9fe99e9d42d8de1a3d9d31a5548f63e16029 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:27:59 +0800 Subject: [PATCH 164/262] feat(app): connect approved sites while the App stays ready --- README.md | 4 +- README.zh-CN.md | 2 +- cmd/piik-app/README.md | 25 ++- cmd/piik-app/README.zh-CN.md | 18 +- .../adr/0010-cross-platform-client-runtime.md | 38 ++-- docs/guide/getting-started.md | 20 +- docs/guide/getting-started.zh-CN.md | 16 +- docs/guide/troubleshooting.md | 10 +- docs/guide/troubleshooting.zh-CN.md | 8 +- docs/project-memory.md | 13 +- .../research/cross-platform-client-runtime.md | 7 +- docs/research/native-client-media.md | 17 +- docs/standards/configuration.md | 32 +-- docs/standards/presentation-lifecycle.md | 12 +- docs/standards/routing-transport.md | 3 +- docs/status.md | 7 +- docs/todo.md | 15 +- internal/app/app.go | 197 +++++++++++------- internal/app/app_test.go | 41 ++-- internal/app/console.go | 2 +- internal/app/launcher/preference.go | 2 +- internal/app/launcher/preference_test.go | 2 - internal/app/launcher/server.go | 134 +++++++++--- internal/app/launcher/server_test.go | 46 ++-- internal/app/launcher/site_access_test.go | 133 ++++++++++++ .../app/loopback/quality_preference_test.go | 2 +- internal/app/loopback/server.go | 166 ++++++++++++--- internal/app/loopback/site_access_test.go | 150 +++++++++++++ site/film/ui/main.tsx | 2 +- site/index.html | 34 ++- src/client/App.tsx | 16 +- .../components/living/AppConnectionButton.tsx | 73 +++++++ src/client/components/living/Header.tsx | 2 + src/client/components/living/LauncherForm.tsx | 88 ++++---- src/client/lib/session.ts | 26 +++ src/client/locales/en.ts | 26 ++- src/client/locales/zh.ts | 26 ++- src/client/native/client.ts | 35 +++- src/client/native/wire.ts | 5 + src/client/pages/AppLauncherPage.tsx | 154 ++++++++++++-- src/client/pages/ControlsPreview.tsx | 4 +- src/client/pages/ViewerPage.tsx | 8 +- src/client/styles.css | 33 ++- tests/client-launch-presentation.test.ts | 22 ++ tests/native-client.test.ts | 41 +++- tests/viewer-native-client-ownership.test.ts | 2 +- 46 files changed, 1318 insertions(+), 401 deletions(-) create mode 100644 internal/app/launcher/site_access_test.go create mode 100644 internal/app/loopback/site_access_test.go create mode 100644 src/client/components/living/AppConnectionButton.tsx diff --git a/README.md b/README.md index 3ab75c0af..57939608d 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,8 @@ device and adjust your voice volume. Piik follows your system language: Chinese or English, with English as the fallback. Use **中 / EN / ✦** in the header to switch language or try the visual mode. -The App offers a local room, a temporary public link, or a connection to your own -site. Keep the App running while sharing through it. +The App can create a local room or temporary public invitation, and stays +available to supply capture for an approved site. Keep it running while sharing. Download a ZIP beginning with **`piik-app`**. The filename also identifies the platform: diff --git a/README.zh-CN.md b/README.zh-CN.md index 8398ca907..781b3e714 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -66,7 +66,7 @@ Piik 是一个免费开源的私密屏幕共享工具,适合游戏直播、一 Piik 默认跟随系统语言,中文系统显示简体中文,其他语言显示英文。 顶部的 **中 / EN / ✦** 可切换中英文或体验纯视觉模式。 -Piik App 提供本地房间、临时公网邀请和连接已有站点三种模式,分享期间需保持 App 运行。 +Piik App 可创建本地房间或临时公网邀请,也能在后台为已授权站点提供采集能力。分享期间需保持 App 运行。 下载以 **`piik-app`** 开头的 ZIP 压缩包,再按文件名中的平台标识选择: diff --git a/cmd/piik-app/README.md b/cmd/piik-app/README.md index 8b54e39e7..72a4bec6c 100644 --- a/cmd/piik-app/README.md +++ b/cmd/piik-app/README.md @@ -24,12 +24,11 @@ publication are separate steps in [deployment](../../docs/deployment.md). ## Modes -- With no mode argument, the App opens a small launcher in the system - Browser. It selects Local, a temporary public HTTPS invitation, or a saved - Site, then enters the normal Host page. -- The launcher remembers the last mode confirmed with **Open Piik** and the saved - Site address. Without a saved mode, it selects Site when an address is saved, - otherwise Public invite; it waits for confirmation before starting. +- With no mode argument, the App opens its launcher and waits for use by an + approved Site. Choose **Local room** or **Public invite** to start a room + service on this device and enter the normal Host page. +- The launcher remembers the last mode confirmed with **Open Piik**. Without a + saved Local/Public choice, it selects Public invite and waits for confirmation. The mode preference is stored beside the settings in `client.json.mode`; [App configuration](../../docs/standards/configuration.md#piik-app-configuration) covers saved preferences and custom paths. @@ -49,14 +48,18 @@ media-server forwarding (SFU) or TURN relay fallback. The For Local invitations, the App automatically selects a sole active address or sole private IPv4. If several addresses remain possible, choose an interface -and IP in the launcher. Public invite and Site mode need no LAN selection. +and IP in the launcher. Public invite and use through a Site need no LAN selection. When bypassing the launcher with `--local`, use `--lan-address
` to resolve an ambiguous choice. The selected address is checked again at startup; it affects Local invitation links, while ICE selects media paths independently. -Site mode uses the configured Site's rooms and any enabled SFU fallback. -An App-opened Site remembers native activation, so later pages there may reuse -the running App. Browser capture remains available without the App; see the +To use a Piik Site, keep the App running and click **App** at the top of the Site. +Confirm its address on the local approval page. The App remembers one approved +Site, which the launcher can open or revoke. Approving another replaces it and +ends the previous Site's App connections; Local rooms remain independent. +The browser remembers activation, so later visits reuse the running App with +that Site's rooms and any enabled SFU fallback. Browser capture remains available +without the App; see the [entry workflow](../../docs/standards/presentation-lifecycle.md#product-surface). The Host page offers Browser and Camera capture alongside available native @@ -111,7 +114,7 @@ For codec-specific failures, see [Windows H264 troubleshooting](../../docs/guide ### Diagnostics -Enable **Debug launch** in the mode selector before opening Piik, reproduce the +Enable **Debug launch** in the launcher before starting a room or connecting a Site, reproduce the problem, then press `D` in the interactive terminal to export a local ZIP. Use `--debug` for failures before the selector opens. Exporting does not stop the share or upload the archive. For Browser diagnostics, click the **Debug** diff --git a/cmd/piik-app/README.zh-CN.md b/cmd/piik-app/README.zh-CN.md index 707768d50..f92de74b9 100644 --- a/cmd/piik-app/README.zh-CN.md +++ b/cmd/piik-app/README.zh-CN.md @@ -21,10 +21,10 @@ Apple 芯片或 Intel Mac,以及 macOS 13 或更新版本。构建程序包与 ## 运行模式 -- 不传模式参数时,App 在系统浏览器中打开启动页。选择本地房间、临时公网 HTTPS 邀请 - 或已保存的站点后,进入通常的房主页面。 -- 启动页会记住上次点击 **进入 Piik** 确认的模式,以及保存的站点地址。 - 没有已保存的模式时,有站点地址就默认选中站点,否则选中公网邀请;确认后才启动。 +- 不传模式参数时,App 打开启动页,等待已授权站点使用。选择 **本地房间** 或 + **公网邀请**,可在这台设备上启动房间服务,进入通常的房主页面。 +- 启动页会记住上次点击 **进入 Piik** 确认的模式。 + 没有已保存的本地/公网选择时,默认选中公网邀请;确认后才启动。 模式偏好保存在配置旁的 `client.json.mode` 中;保存项与自定义路径见 [App 配置](../../docs/standards/configuration.md#piik-app-configuration)。 - `--site`、`--local` 和 `--link` 用于在 CI 或开发时直接选择模式。 @@ -39,12 +39,14 @@ Apple 芯片或 Intel Mac,以及 macOS 13 或更新版本。构建程序包与 [路由参考](../../docs/standards/routing-transport.md)。 本地邀请会自动选择唯一的活动地址或唯一的私有 IPv4 地址。仍有多个可能地址时, -请在启动页选择网卡和 IP。公网邀请和站点模式无需选择局域网地址。 +请在启动页选择网卡和 IP。公网邀请和搭配站点使用时无需选择局域网地址。 通过 `--local` 跳过启动页时,可用 `--lan-address
` 指定地址来解决多地址歧义。 App 会在启动时再次检查所选地址;它只影响本地邀请链接,媒体路径仍由 ICE 独立选择。 -站点模式使用所配置站点的房间,以及该站点已启用的 SFU 兜底。 -通过 App 打开的站点会记住原生功能的启用选择,之后打开该站点的页面可继续使用正在运行的 App。 +使用 Piik 站点时,保持 App 运行,点击站点顶部的 **App**,在本机确认页核对网址并允许使用。 +App 记住一个已授权站点,可在启动页打开或取消授权。允许另一个站点会替换旧站点, +结束旧站点的 App 连接;本地房间不受影响。同一浏览器记住启用选择, +之后访问站点即可继续使用 App,同时使用该站点的房间和已启用的 SFU 兜底。 没有 App 时仍可使用浏览器采集,详见[进入流程](../../docs/standards/presentation-lifecycle.md#product-surface)。 房主页面提供浏览器、摄像头采集,以及可用的原生窗口或屏幕,请明确选择来源。 @@ -87,7 +89,7 @@ App 在独立 Windows 控制台中启动时,启动或运行错误会保留在 ### 诊断 -在启动页开启 **诊断启动**,进入 Piik 并复现问题,再在交互终端按 `D` 导出本地 ZIP。 +在启动房间或连接站点前,先在启动页开启 **诊断启动**,复现问题后在交互终端按 `D` 导出本地 ZIP。 启动页出现前就失败时,使用 `--debug`。导出不会停止分享或上传文件。 浏览器诊断通过主题按钮后的 **诊断** 芯片图标开启:确认重新加载后,使用 **网页报告** 下载。 调查浏览器与 App 配合的问题时,请提供同一次复现的两份报告。 diff --git a/docs/adr/0010-cross-platform-client-runtime.md b/docs/adr/0010-cross-platform-client-runtime.md index 0a5b17974..810af173d 100644 --- a/docs/adr/0010-cross-platform-client-runtime.md +++ b/docs/adr/0010-cross-platform-client-runtime.md @@ -46,7 +46,7 @@ topologies and makes a saved Site unavailable while another room source runs. public process identity, not authentication. The listener accepts the current Local Host origin and the one user-saved Site origin. Origin and Host validation plus the Browser's local-network permission own this boundary. -4. Loopback v9 starts with a strict `hello` handshake. Up to two participant +4. Native control starts with a strict `hello` handshake. Up to two participant control sessions may coexist; each retains its own share and media lifetime. This bounds concurrent Native work, not the shared server's room count. Ending one control session cannot end another; App exit waits for all @@ -69,11 +69,15 @@ topologies and makes a saved Site unavailable while another room source runs. wrapper or UI. 6. The system Browser remains the UI. Browser extensions, userscripts, Electron, Tauri, and resident services need new evidence before they can replace this - smaller boundary. An App-opened Site stores a non-secret opt-in at that - exact Browser origin, so later manually opened pages may discover the running - App. Pages without that opt-in do not probe localhost or request local- - network permission. Clearing Site data simply requires opening it from the - App again. + smaller boundary. An App-opened or explicitly connected Site stores a + non-secret opt-in at that exact Browser origin, so later pages may discover + the running App. Without this opt-in, only the user's App connection action + starts discovery. Its public loopback discovery returns process identity and + a local approval URL, never native capability, source or preference data. + The App-owned approval page displays the exact requesting origin and requires + confirmation before saving it. It rejects cross-origin mutations and framing; + the Site cannot approve itself. A popup message requests an authorized health + recheck, not authorization. Clearing Site data requires connecting again. 7. A self-contained Local deployment serves reachable LAN peers without a central Piik service. Its explicit `--link` mode starts one accountless Cloudflare Quick Tunnel for the same HTTP/WebSocket surface. The App @@ -130,12 +134,12 @@ topologies and makes a saved Site unavailable while another room source runs. extra threshold, timer, score, route operation, or representation ladder is added. Only a consumed App-launch marker travels in the URL fragment. Exact target identity travels over loopback. -9. The App uses the system Browser as its only UI. On every launch, the current - lightweight control center offers Local, public link, and Site. The Site value - is stored in App configuration and remains one click on later launches; - selecting Local or public link does not disable background RPC access for the - saved Site. Command-line mode - selectors remain automation inputs rather than the normal interface. An +9. The App uses the system Browser as its only UI. It remains available to the + saved Site before a room mode is chosen. Its persistent launcher offers Local + and public link, shows the approved Site and allows revocation. Starting a + Local/public room does not disable that Site's native access or approval UI; + neither visiting the launcher nor approving a Site creates a room. Command-line + entry selectors remain automation inputs rather than the normal interface. An embedded shell requires a reproduced product failure and one comparative decision. 10. Local mode is one explicit server composition: static current assets, @@ -143,7 +147,7 @@ topologies and makes a saved Site unavailable while another room source runs. current LAN IPv4 origins. It uses no public discovery by default. Public-link mode adds its temporary HTTPS origin and the bounded public STUN discovery and survey configuration in [ADR-0009](0009-optional-nat-prediction.md). - Site mode consumes that Site's + A participant on an approved Site consumes that Site's configured STUN survey. Both feed the same connection-local prediction adapter; Native additionally owns its UDP socket and best-effort port mapping. 11. One App configuration owns the optional Site origin and an optional, @@ -152,7 +156,12 @@ topologies and makes a saved Site unavailable while another room source runs. App bootstraps a selected page through a fragment consumed before authentication; the page retains only the non-secret App opt-in at its origin. Friends use the existing room invitation grant. No App-specific - room authorization system is added. + room authorization system is added. Site replacement/revocation uses this + same configuration owner, immediately denies new control requests from the + old origin and cancels its existing sessions. Retirement is bounded and does + not wait for unrelated Local sessions. Cleanup failure never restores the + revoked permission; concurrent approval pages must review a changed Site + before replacing it. 12. Local authority shutdown first ends every in-memory room through the current `room-closed` path, then closes signaling and HTTP. The App cancels the server context, calls that end-then-close sequence under one bounded @@ -231,6 +240,7 @@ architectural claim. - [Cloudflare Tunnel WebSocket support](https://developers.cloudflare.com/cloudflare-one/faq/cloudflare-tunnels-faq/) - [RFC 6455 WebSocket Origin model](https://www.rfc-editor.org/rfc/rfc6455.html) - [Chrome Local Network Access](https://developer.chrome.com/blog/local-network-access) +- [Browser popup message origin/source checks](https://developer.mozilla.org/en-US/docs/Web/API/Window/postMessage#security_concerns) - [Discord local RPC](https://docs.discord.com/developers/topics/rpc) - [OBS WebSocket protocol](https://github.com/obsproject/obs-websocket/blob/master/docs/generated/protocol.md) - [LocalSend protocol](https://github.com/localsend/protocol/blob/main/README.md) diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index f6d2e6265..97006eb74 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -100,20 +100,20 @@ See the [App guide](../../cmd/piik-app/README.md) for package and runtime detail | --- | --- | --- | | **Local room** | Friends are on the same local network. | If several entries appear under **Local invitation address**, choose the interface and IP on your friends' network. Your computer must be reachable from theirs. A local site passphrase is optional. | | **Public invite** | Friends are elsewhere and you want a temporary invitation. | Needs Internet access. The public address lasts for this App run. | -| **Connect to Site** | Your group already has a Piik site. | Enter its address; the App adds capture capabilities to the site's sharing page. | For Local room and Public invite, expand **Site passphrase** to set an optional access passphrase. -For **Connect to Site**, enter a domain such as `demo.piik.tv` or paste the full -site address, then choose **Open Piik**. HTTPS is added when omitted; for an HTTP -site, include `http://`. Enter the site's passphrase on the page that opens if asked. -After opening the site through the App once, the same browser remembers that -entry. Keep the App running with that site configured and you can visit the -address directly. **Start sharing** opens the selector; choose **Apps / Windows** -or **Screens** for App capture. Allow local-network access if the browser asks. -In another browser, in a private window, or after clearing site data, enter through -the App again. For a different site, update the address in the App first. +**Already have a Piik site?** Keep the App running, open the site and click +**App** at the top. Check the address on the local confirmation page, then choose +**Allow access**. Enter any site passphrase on the site itself. Allow local-network +access if the browser asks. No room mode needs to be selected in the launcher. + +The same browser remembers this connection. On later visits, **Start sharing** +offers **Apps / Windows** and **Screens** from the running App. In another browser, +in a private window or after clearing site data, connect again. The launcher +shows the approved site and lets you revoke it. Approving a different site +replaces the previous one and ends that site's App connections. **A page that opens does not guarantee a video connection.** Public invite gives your room a temporary web address; picture and sound still travel between diff --git a/docs/guide/getting-started.zh-CN.md b/docs/guide/getting-started.zh-CN.md index b26064059..540ce8e81 100644 --- a/docs/guide/getting-started.zh-CN.md +++ b/docs/guide/getting-started.zh-CN.md @@ -91,17 +91,17 @@ Linux 原生采集还需要[系统组件](../../native/capture/linux/README.md) | --- | --- | --- | | **本地房间** | 朋友和你在同一个局域网。 | 出现多个「局域网邀请地址」时,选择与朋友同一网络的网卡和 IP。对方需要能连接到你的电脑,可以设置本地站点口令。 | | **公网邀请** | 朋友在别处,想临时发个邀请。 | 需要互联网连接;公网地址只在本次 App 运行期间有效。 | -| **连接站点** | 你们已经有一个 Piik 站点。 | 输入站点地址,在原有分享页面使用 App 的采集能力。 | 本地房间和公网邀请可展开 **站点口令** 设置可选的访问口令。 -使用 **连接站点** 时,输入域名(例如 `demo.piik.tv`)或粘贴完整网址,然后点击 **进入 Piik**。 -省略前缀时自动使用 HTTPS;HTTP 站点请填写 `http://`。 -如果站点要求访问口令,在打开的站点页面输入。通过 App 打开过一次后,同一浏览器会记住这个入口。 -保持 App 运行并使用该站点配置,之后就能直接访问网址。点击 **开始分享**, -在选择器里选 **程序 / 窗口** 或 **屏幕**,即可使用 App 采集。 -浏览器询问本地网络权限时,请允许访问。换浏览器、使用无痕窗口或清除站点数据后, -需要重新从 App 进入一次;换用其他站点时,先在 App 中更新地址。 +**已经有 Piik 站点?** 保持 App 运行,打开站点,点击顶部的 **App**。 +在本机确认页核对网址,点击 **允许使用**。如果站点要求口令,在站点页面输入。 +浏览器询问本地网络权限时,请允许访问。启动页无需选择房间模式。 + +同一浏览器会记住这次连接。之后直接访问站点,点击 **开始分享**, +选择 **程序 / 窗口** 或 **屏幕**,即可使用正在运行的 App。 +换浏览器、使用无痕窗口或清除站点数据后,重新连接即可。 +启动页显示已授权的站点,也可以取消授权;允许新站点会替换旧站点,并结束旧站点的 App 连接。 **网页能打开,不代表视频一定能连通。** 公网邀请为房间提供临时网页地址, 画面和声音仍靠参与者之间的连接传输。此模式没有媒体服务器兜底, diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 3ee5e3f02..958201ccd 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -43,7 +43,7 @@ checks, a copyable failure summary and browser diagnostics. The summary contains the Web version, message and error code, without invitation links or source names. - **App disconnected or request timed out:** check that the App is running and - responding, open this site from the App, then refresh. Allow local network + responding, then click **App** at the top of the site to connect. Allow local network access if the browser asks. A timeout alone does not establish a graphics fault. - **App could not start the selected source:** try an ordinary window or screen, then compare with the **Browser** source picker. On Windows, try **Auto** or @@ -83,9 +83,9 @@ See [Server deployment](../operations/self-hosting.md) for the configuration. ## App windows or screens are missing -Keep Piik App running on the computer where you are sharing. Open the current -site from the App; for a hosted site, use the App's **Site** mode with that -site's address. If the browser asks for local-network access, allow it, then +Keep Piik App running on the computer where you are sharing. Click **App** at +the top of the site, then confirm its address on the local approval page. +If the browser asks for local-network access, allow it, then refresh the source list. If access was previously blocked, review this site's browser permissions first. @@ -93,7 +93,7 @@ The source picker distinguishes these outcomes: | Message | Next step | | --- | --- | -| Cannot connect to Piik App | Confirm the App is running and opened this site. Close unused Piik tabs using the App, then refresh the list; up to two pages can use the App's native capture or viewing at once. | +| Cannot connect to Piik App | Confirm the App is running and this site is approved. Close unused Piik tabs using the App, then refresh the list; up to two pages can use the App's native capture or viewing at once. | | Piik App and this page are incompatible | Update Piik App and reload the page. | | Piik App capture is currently unavailable | Check the App diagnostics for capture or encoder availability. Review your encoding setting or use browser capture. | | Could not read the screen and window list | Refresh to retry. If it keeps failing, export both the App and browser Debug reports. | diff --git a/docs/guide/troubleshooting.zh-CN.md b/docs/guide/troubleshooting.zh-CN.md index fa2db54f3..cef7570bf 100644 --- a/docs/guide/troubleshooting.zh-CN.md +++ b/docs/guide/troubleshooting.zh-CN.md @@ -41,7 +41,7 @@ 电视下方的错误图标说明当前失败;点击可打开排查面板,查看建议、复制失败信息, 或开启网页诊断。复制内容包含网页版本、提示和错误编号,不包含邀请链接或来源名称。 -- **App 连接断开或请求超时**:确认 App 仍在运行且可以响应,从 App 打开当前站点后刷新。 +- **App 连接断开或请求超时**:确认 App 仍在运行且可以响应,点击站点顶部的 **App** 完成连接。 浏览器询问本地网络权限时允许访问。超时只说明请求未及时完成,不能据此认定显卡故障。 - **App 未能启动所选来源**:先换一个普通窗口或屏幕;再对照「浏览器」选源。 Windows 上可试「自动」或 VP8,后续按[显卡排查](#windows-上-h264-分享失败)操作。 @@ -72,15 +72,15 @@ ## App 找不到窗口或屏幕 -在分享画面的这台电脑上保持 Piik App 运行,并从 App 打开当前站点。 -使用已部署的站点时,在 App 的 **站点** 模式中填写该站点地址。 +在分享画面的这台电脑上保持 Piik App 运行,点击站点顶部的 **App**, +在本机确认页核对网址并允许使用。 浏览器询问本地网络权限时允许访问,再刷新来源列表;如果之前拒绝过,先检查当前站点的浏览器权限。 选源器会区分以下情况: | 提示 | 处理方法 | | --- | --- | -| 未能连接 Piik App | 确认 App 正在运行,且通过 App 打开了当前站点。关闭其他正在使用 App 的闲置 Piik 标签页后刷新列表;最多两个页面可以同时使用 App 的原生采集或收看功能。 | +| 未能连接 Piik App | 确认 App 正在运行,且当前站点已获得授权。关闭其他正在使用 App 的闲置 Piik 标签页后刷新列表;最多两个页面可以同时使用 App 的原生采集或收看功能。 | | Piik App 与当前页面不兼容 | 更新 Piik App 并刷新页面。 | | Piik App 采集功能当前不可用 | 查看 App 诊断信息中的采集和编码能力,检查编码设置,或使用浏览器采集。 | | 未能读取 App 的窗口和屏幕列表 | 刷新重试;若持续失败,请导出同一次尝试的 App 和网页诊断报告。 | diff --git a/docs/project-memory.md b/docs/project-memory.md index 4ca8d2b6a..577a65887 100644 --- a/docs/project-memory.md +++ b/docs/project-memory.md @@ -1,6 +1,6 @@ # Project Memory -Last updated: 2026-10-09 +Last updated: 2026-10-11 Piik is private, low-latency screen sharing for one Host and up to 20 authenticated friends. The current product surface is Web Host, Web Viewer, and @@ -9,11 +9,12 @@ mobile Browsers can also host. It is not a public broadcast service. Hosted and App run one shared Go core ([ADR-0012](./adr/0012-shared-go-backend-core.md)); Node and Vite build the Browser UI and are not a runtime. Source includes a cross-platform App that -runs that same core in a self-contained Local deployment or opens one configured -Site through the system Browser. Its single Go process presents Local, temporary -public-link, and saved-Site choices on every launch, starts one process-level -loopback capability service, owns the Local room authority in-process when -selected, and owns the explicit native media path chosen in the same Web UI. +runs that same core in a self-contained Local deployment or supplies native +capability to one approved Site through the system Browser. Its single Go +process starts the loopback capability service before room selection. The +persistent launcher offers Local and temporary public-link choices and manages +the Site's explicit local approval. Site use can remain on standby without a +Local room service; native media still belongs to the participant's Web UI. Room source and media implementation are independent: a topology may mix Browser and Native peers without changing participant, signaling, capacity, or route identity. Local is LAN-first; explicit one-link mode exposes that same diff --git a/docs/research/cross-platform-client-runtime.md b/docs/research/cross-platform-client-runtime.md index 125dc0a74..58aa94661 100644 --- a/docs/research/cross-platform-client-runtime.md +++ b/docs/research/cross-platform-client-runtime.md @@ -53,9 +53,10 @@ The App and Hosted Piik run the same server core, so the local package is one executable carrying the built Browser assets plus the optional capture and tunnel sidecars; [the App README](../../cmd/piik-app/README.md) owns that layout. In Local mode the App runs the server in its own process with -explicit local configuration. In Site mode it starts no server, and the system -Browser opens the saved Site while the same process remains its loopback -native-media owner. The system Browser remains the UI. +explicit local configuration. While waiting for an approved Site it starts no +room server; the same process remains the Site's loopback native-media owner. +The persistent launcher owns local approval and room startup independently. +The system Browser remains the UI. Official Go process APIs require every started child to be waited and provide a bounded `WaitDelay` for cancellation and stuck I/O. That guidance now covers the diff --git a/docs/research/native-client-media.md b/docs/research/native-client-media.md index 9b9fdefa1..d32fd84c4 100644 --- a/docs/research/native-client-media.md +++ b/docs/research/native-client-media.md @@ -3,8 +3,8 @@ - Reviewed: 2026-10-11 - Scope: platform capture, shared encoded sources, Pion transport and Browser decode; current behavior belongs to [media quality](../standards/media-quality.md) -- Status: Windows physical native Host and Viewer gates passed; current macOS - changes await SDK compilation, and macOS/Linux physical media gates remain open +- Status: Windows physical native Host and Viewer gates passed; both macOS + architectures compile and pass packaged startup; macOS/Linux physical media gates remain open ## Windows Audio Exclusion @@ -558,11 +558,12 @@ replaces only its compression session when the fitted size changes. The frame envelope carries actual dimensions; declared output limits and room/route identity remain unchanged. Encoded-input derivation uses the same worker. -This implementation has not yet been compiled with the macOS SDK or executed on -macOS. The existing native check now includes content-geometry validation and -extends hardware self-testing across successive source shapes. Run those checks -on Apple silicon and Intel, then verify real window resizing, Retina transitions, -cropped image fidelity and quiet-source recovery before accepting this boundary. +The candidate passes macOS SDK compilation and packaged startup on Apple silicon +and Intel in [CI run 38128668158](https://github.com/TNTcraftHIM/Piik/actions/runs/38128668158). +The native check includes content-geometry validation and successive-shape +hardware self-tests when an encoder is available. Runner checks do not replace +real window resizing, Retina transitions, cropped image fidelity or quiet-source +recovery on a physical Mac. Local synthetic-input controls exercised that production worker through four successive source sizes: 1080x1920, 3440x1440, 640x480 and 1920x1080. Under the @@ -645,7 +646,7 @@ coded storage. Windows copies the aperture while retaining the decoder surface, so downstream outputs receive the visible picture. The native regression checks right/bottom H.264 padding with real software decoding, visible edge pixels, and an offset NV12 aperture through the DXGI-buffer path. Oversized coded/reference -storage remains rejected; the macOS SDK boundary above still applies. +storage remains rejected; the physical macOS boundary above still applies. An additional nonzero left/top SPS crop exposed a local Media Foundation software decoder limit: its raw output omitted right/bottom picture pixels before Piik's diff --git a/docs/standards/configuration.md b/docs/standards/configuration.md index 23afca7a2..5f657105f 100644 --- a/docs/standards/configuration.md +++ b/docs/standards/configuration.md @@ -108,30 +108,32 @@ schema field. The user settings are: | Setting | Contract | | --- | --- | -| `site` | Saved Piik Site origin. The launcher or `--site` updates it. | +| `site` | One approved Piik Site origin. Local confirmation or explicit `--site` updates it; the launcher can revoke it. | | `localAccessPassword` | Empty by default. Optional password for the App's Local room authority, with the same exact-match behavior as `SITE_ACCESS_PASSWORD`. It is separate from a hosted site's password. | -Site input in the launcher and `--site` defaults to HTTPS when the scheme is -omitted; explicit HTTP or HTTPS is preserved. The App saves the normalized origin -and still rejects credentials, room paths, queries and fragments. It does not -probe protocols or downgrade a failed HTTPS connection to HTTP. +The Site's App button requests local confirmation for the page's exact origin. +The `--site` automation input defaults to HTTPS when the scheme is omitted; +explicit HTTP or HTTPS is preserved. Both save the normalized origin and reject +credentials, room paths, queries and fragments. The App does not probe protocols +or downgrade a failed HTTPS connection to HTTP. The launcher remembers the last mode confirmed with the launch button. This optional preference lives beside the configuration as `client.json.mode` (or `.mode`); removing it restores the initial selection without -changing saved settings. Older App versions ignore this file. Without a valid -preference, a saved Site is selected when present; otherwise the public +changing saved settings. Without a valid Local/Public preference, the public invitation link is selected. Choosing a mode does not start it until confirmed. Local mode remains available for the same network. Selecting Local or Public Link, including through the CLI, preserves the saved -Site and its allowed App origin; only an explicit Site edit replaces it. +Site and its allowed App origin; only local approval/revocation or `--site` +changes it. The Site can use native capabilities while the App waits in its +launcher; it does not need a Local room service. For Local mode, the launcher selects a sole active address or sole private IPv4 address automatically. With several choices it shows interface names and IPs; an ambiguous choice must be selected before launch. `--lan-address` preselects an active address. Selection applies to this launch and is revalidated at startup. It sets the local invitation origin; HTTP still listens on the existing wildcard -listener and ICE remains free to use available media interfaces. Public Link and -Site mode require no local address selection. +listener and ICE remains free to use available media interfaces. Public Link +and use through an approved Site require no local address selection. Command-line options select entry and local runtime behavior: @@ -165,10 +167,12 @@ Diagnostics are local and opt-in. Enable them **before** reproducing the problem | Browser Host/Viewer | Click the **Debug** chip icon after the theme control and confirm the reload, or add `?debug=1` before any invitation fragment | The same control becomes a download arrow for the web report | | Hosted Server | Start with `--debug`, `PIIK_DEBUG=server` or `PIIK_DEBUG=route` | On Unix, `kill -USR1 `; also exported at orderly shutdown | -App **Debug launch** enables App and Browser collection for that run before -starting the selected mode. It records Native capability results and subsequent -capture, connection and local-server activity. Use `--debug` for failures before -the mode selector opens. This choice does not change the saved App configuration. +App **Debug launch** starts App collection immediately, including while waiting +for a Site. Pages subsequently opened from the launcher also collect Browser +logs; already-open Site pages use their own Debug control. It records Native +capability results and subsequent capture, connection and local-server activity. +Use `--debug` for failures before the launcher opens. This choice does not change +the saved App configuration. When bypassing the mode selector, `--debug` and `PIIK_DEBUG=client` enable backend collection; enable the Browser control separately for browser-side diagnosis. diff --git a/docs/standards/presentation-lifecycle.md b/docs/standards/presentation-lifecycle.md index ab386115d..72a86bd9b 100644 --- a/docs/standards/presentation-lifecycle.md +++ b/docs/standards/presentation-lifecycle.md @@ -34,10 +34,14 @@ is absent, provided the room service and their admitted sessions remain availabl [Rooms and access](./rooms-access.md#room-interactions) owns interaction authority and retention; [status](../status.md#accepted-interaction-phase) owns release readiness. -Piik App opens this same application in the system Browser. Its small -startup surface selects Local, temporary public invitation, or a configured -Site before entering the Host workspace. Viewer links opened at that activated -origin may use the same running App without changing the Viewer UI. Starting a share offers the +Piik App opens this same application in the system Browser. Its startup surface +offers Local and temporary public invitation; the running App is already +available to its approved Site before either mode starts. A Site's explicit App +action discovers the local approval page, where the user confirms its exact +origin. The launcher shows and revokes that approval. Activating a page supplies +native capability without restarting its room or healthy media session. +Viewer links opened at that activated origin may use the same running App +without changing the Viewer UI. Starting a share offers the Browser picker and any exact App-owned native windows. It never chooses a window automatically. A reproduced Browser-window, capture, or background failure is required before introducing an embedded Web runtime. diff --git a/docs/standards/routing-transport.md b/docs/standards/routing-transport.md index 1044a7af3..87dccb742 100644 --- a/docs/standards/routing-transport.md +++ b/docs/standards/routing-transport.md @@ -272,7 +272,8 @@ design. All-UDP-blocked networks currently end in bounded failure. App Local mode runs this graph without SFU, NAT prediction, or default STUN. One-link mode tunnels only HTTP/WebSocket control and adds public STUN to the same Browser/Native P2P edges. Reachable peers form one mixed relay tree; an -unreachable path adds no route type or score. Site mode uses Site transport. +unreachable path adds no route type or score. App-enabled Site pages use that +Site's transport configuration. A Native Host reserves one loopback edge for Browser preview; it consumes no route-copy capacity or gateway mapping. Direct children and the admitted Native diff --git a/docs/status.md b/docs/status.md index a16f778ef..35d2a0bd3 100644 --- a/docs/status.md +++ b/docs/status.md @@ -72,8 +72,8 @@ the three recommended presets. Windows and Linux output preserve source shape within the selected ceiling. The [media assessment](./research/native-client-media.md#media-capability-extension-assessment) owns local codec, derived-output, SFU, Browser/Native fanout and live-control -evidence. The macOS geometry implementation still needs SDK compilation and -hardware verification; cross-browser/device and route-change acceptance remain +evidence. Both macOS architectures pass SDK compilation and packaged startup; +physical geometry verification, cross-device and route-change acceptance remain open. Additional codecs are not part of this candidate. Magicsock adoption is no-go for this candidate: its App-to-App augmentation does not improve the core App-to-Browser use case. The @@ -81,6 +81,9 @@ not improve the core App-to-Browser use case. The owns the retained experiment and product decision. Its package is retired; independent ordinary ICE and media repairs remain in the candidate. Publication remains subject to final acceptance. +The candidate App now waits for approved Sites independently of Local/public +room startup. Its two-choice launcher retains one exact Site approval and +revocation, using the existing loopback and native-session owners. Immutable release artifacts own delivery completion; [TODO](./todo.md#now) owns remaining work. diff --git a/docs/todo.md b/docs/todo.md index 3a0cc64a9..17b90c791 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -84,8 +84,9 @@ Implementation needs a concrete useful result within the product model. Remaining receiver acceptance needs physical limited decoders and camera orientation behavior, beyond the bounded SDP-limit and mixed-route controls. Browser display capture, Browser relay, Browser-to-Native fanout and SFU have - bounded high-resolution continuity evidence. macOS geometry SDK/hardware - checks and physical 4K/game capture remain open. The fanout controls did not + bounded high-resolution continuity evidence. macOS geometry hardware + checks and physical 4K/game capture remain open; both macOS package targets + now pass SDK compilation and startup. The fanout controls did not establish sustained 60 fps; retain that performance limit when arranging device/network acceptance. Preserve safety limits and framework-owned adaptation. The @@ -115,11 +116,11 @@ Implementation needs a concrete useful result within the product model. existing device selector. Keep it outside this candidate until a concrete use case justifies that boundary and physical latency/echo/background checks. This proposal does not reopen room voice. -- [ ] **Passive App attachment: design hold.** Site mode authorizes one selected - origin and supplies native media without starting a local room server. A - passive replacement needs an accepted site-consent/discovery flow; it must not - admit arbitrary sites or add another runtime owner. Keep Site mode until that - decision; removal of the Demo prefill does not authorize changing site trust. +- [ ] **Passive App attachment acceptance.** The two-choice launcher and explicit + local Site approval replace the third mode, using the existing native runtime. + Include first connection, saved approval, replacement/revocation and Local + coexistence in final packaged acceptance. Browser permission and cross-platform + checks must not be inferred from the local development preview. ## Awaiting Device Or Reporter Evidence diff --git a/internal/app/app.go b/internal/app/app.go index a9bd6859a..8cac1855c 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "strings" + "sync" "syscall" "time" @@ -77,9 +78,12 @@ func Run(ctx context.Context, options Options) (returnedErr error) { options.Debug = options.Debug || debugEnabled(os.Getenv("PIIK_DEBUG")) options.console = newConsole(cancel, options.DisableBrowser) var recorder *diagnostics.Recorder + var diagnosticsMu sync.Mutex var restoreLogger func() defer func() { cancel() + diagnosticsMu.Lock() + defer diagnosticsMu.Unlock() returnedErr = finishRun(options.console, recorder, restoreLogger, returnedErr) }() enableDiagnostics := func() error { @@ -181,19 +185,24 @@ func Run(ctx context.Context, options Options) (returnedErr error) { return err } } - if !explicitMode(options) { - return runLauncher(ctx, options, configPath, config, control, func(selected *Options, config appconfig.Config) error { - if selected.Debug { - if err := enableDiagnostics(); err != nil { - return err - } - selected.logger = recorder.Logger() + return runLauncher(ctx, options, configPath, config, control, func(selected *Options, config appconfig.Config) error { + diagnosticsMu.Lock() + defer diagnosticsMu.Unlock() + if ctx.Err() != nil { + return ctx.Err() + } + if selected.Debug { + if err := enableDiagnostics(); err != nil { + return err } - recordContext(*selected, config) - return nil - }) - } - return runConfigured(ctx, options, config, control) + } + if recorder != nil { + selected.Debug = true + selected.logger = recorder.Logger() + } + recordContext(*selected, config) + return nil + }) } func finishRun(console *console, recorder *diagnostics.Recorder, restoreLogger func(), err error) error { @@ -218,18 +227,6 @@ func explicitMode(options Options) bool { return options.SiteSet || options.Local || options.Link || options.DisableBrowser } -func runConfigured( - ctx context.Context, - options Options, - config appconfig.Config, - control *loopback.Server, -) error { - if config.Site != "" && !options.Local && !options.Link { - return runSite(config.Site, options, control) - } - return runLocal(ctx, options, config, control) -} - func runLauncher( ctx context.Context, options Options, @@ -238,10 +235,29 @@ func runLauncher( control *loopback.Server, configureDiagnostics func(*Options, appconfig.Config) error, ) (returnedErr error) { + automatic := explicitMode(options) + initialOptions := options + initialConfig := config mode, err := launcher.LoadMode(configPath) if err != nil { slog.Debug("Could not restore the last launcher mode", diagnostics.Error(err)) } + // Site approval and room startup mutate separate fields of one config. + // Serialize persistence and publish only the complete successfully saved value. + var configMu sync.Mutex + launcherOrigin := "" + updateConfig := func(change func(*appconfig.Config)) (appconfig.Config, error) { + configMu.Lock() + defer configMu.Unlock() + next := config + change(&next) + if err := appconfig.Save(configPath, next); err != nil { + return config, err + } + config = next + err := control.SetAllowedOrigins(append(allowedOrigins(config.Site, initialOptions.Port), launcherOrigin)) + return config, err + } launch, err := launcher.Start( ctx, webassets.FS(), @@ -250,69 +266,112 @@ func runLauncher( Site: config.Site, Version: buildVersion(), Revision: BuildRevision, LocalAccessPassword: config.LocalAccessPassword, Debug: options.Debug, LANAddress: options.LANAddress, + ChangeSite: func(site string) (string, error) { + current, err := updateConfig(func(next *appconfig.Config) { next.Site = site }) + return current.Site, err + }, + EnableDebug: func() error { + configMu.Lock() + defer configMu.Unlock() + selected := initialOptions + selected.Debug = true + return configureDiagnostics(&selected, config) + }, }, ) if err != nil { return err } - defer launch.Close() + defer func() { returnedErr = errors.Join(returnedErr, launch.Close()) }() + control.SetConnectURL(launch.URL()) + defer control.SetConnectURL("") launchAddress, err := url.Parse(launch.URL()) if err != nil { return err } - control.SetAllowedOrigins(append( - allowedOrigins(config.Site, options.Port), - launchAddress.Scheme+"://"+launchAddress.Host, - )) - options.console.show(consoleView{state: "setup", entry: launch.URL()}) - options.console.openBrowser(launch.URL()) + configMu.Lock() + launcherOrigin = launchAddress.Scheme + "://" + launchAddress.Host + err = control.SetAllowedOrigins(append(allowedOrigins(config.Site, options.Port), launcherOrigin)) + configMu.Unlock() + if err != nil { + return err + } + if !automatic { + options.console.show(consoleView{state: "setup", entry: launch.URL()}) + options.console.openBrowser(launch.URL()) + } else if initialConfig.Site != "" && !options.Local && !options.Link { + // --site is still an explicit approval shortcut. It supplies native + // capabilities immediately; the launcher can start a Local room later. + openSite(initialConfig.Site, options) + } var selection launcher.Selection - select { - case selection = <-launch.Selection(): - case <-ctx.Done(): - return nil - case err = <-launch.Done(): - if err != nil { - return fmt.Errorf("Piik App launcher stopped unexpectedly: %w", err) + if automatic && (options.Local || options.Link || initialConfig.Site == "") { + selection = launcher.Selection{Mode: launcher.ModeLocal, LocalAccessPassword: initialConfig.LocalAccessPassword} + if options.Link { + selection.Mode = launcher.ModeLink } - return nil + launch.Begin() + } else { + select { + case selection = <-launch.Selection(): + case <-ctx.Done(): + return nil + case err = <-control.Done(): + return err + case err = <-launch.Done(): + if err != nil { + return fmt.Errorf("Piik App launcher stopped unexpectedly: %w", err) + } + return nil + } + } + if selection.Language != "" { + options.console.setLanguage(selection.Language) } - options.console.setLanguage(selection.Language) if err := launcher.SaveMode(configPath, selection.Mode); err != nil { slog.Debug("Could not save the launcher mode", diagnostics.Error(err)) } - if selection.Mode == launcher.ModeSite { - config.Site = selection.Site - } else { - config.LocalAccessPassword = selection.LocalAccessPassword - } options.Local = selection.Mode == launcher.ModeLocal options.Link = selection.Mode == launcher.ModeLink options.Debug = options.Debug || selection.Debug if selection.LANAddress != nil { options.LANAddress = *selection.LANAddress } - if err = configureDiagnostics(&options, config); err != nil { + selectedConfig, err := updateConfig(func(next *appconfig.Config) { + next.LocalAccessPassword = selection.LocalAccessPassword + }) + if err != nil { + err = fmt.Errorf("Piik App configuration could not be saved: %w", err) launch.SetResult("", err) - <-launch.Handled() + if !automatic { + <-launch.Handled() + } return err } - if err = appconfig.Save(configPath, config); err != nil { - err = fmt.Errorf("Piik App configuration could not be saved: %w", err) + if err = configureDiagnostics(&options, selectedConfig); err != nil { launch.SetResult("", err) - <-launch.Handled() + if !automatic { + <-launch.Handled() + } return err } - control.SetAllowedOrigins(allowedOrigins(config.Site, options.Port)) - options.DisableBrowser = true + if !automatic { + options.DisableBrowser = true + } ready := make(chan string, 1) - options.Ready = func(target string) { ready <- target } + onReady := options.Ready + options.Ready = func(target string) { + if onReady != nil { + onReady(target) + } + ready <- target + } runtimeContext, cancelRuntime := context.WithCancel(ctx) runtimeDone := make(chan struct{}) var runtimeErr error go func() { - runtimeErr = runConfigured(runtimeContext, options, config, control) + runtimeErr = runLocal(runtimeContext, options, selectedConfig, control) close(runtimeDone) }() defer func() { @@ -326,24 +385,20 @@ func runLauncher( launch.SetResult(target, nil) case <-runtimeDone: launch.SetResult("", runtimeErr) - <-launch.Handled() + if !automatic { + <-launch.Handled() + } return nil case <-ctx.Done(): launch.SetResult("", ctx.Err()) return nil } - select { - case <-launch.Handled(): - _ = launch.Close() - case <-runtimeDone: - return nil - case <-ctx.Done(): - return nil - } select { case <-runtimeDone: return nil + case err = <-launch.Done(): + return err case <-ctx.Done(): return nil } @@ -401,11 +456,7 @@ func openDiagnostics(directory string) (*diagnostics.Recorder, error) { return diagnostics.Open(filepath.Join(cache, "Piik", "logs"), "client", BuildRevision) } -// runSite opens the configured Piik Site in the Browser and waits for the -// loopback server, which is the only thing this mode owns. It takes no context: -// cancellation reaches it through control.Done(). -func runSite(site string, options Options, control *loopback.Server) error { - var err error +func openSite(site string, options Options) { slog.Debug("piik-client", "event", "mode", "mode", "site") targetURL := launchURL(site) view := consoleView{mode: "site", state: "ready", entry: targetURL} @@ -415,14 +466,11 @@ func runSite(site string, options Options, control *loopback.Server) error { } if !options.DisableBrowser { options.console.openBrowser(targetURL) - } else if options.Ready != nil { + } + if options.Ready != nil { options.Ready(targetURL) } slog.Debug("piik-client", "event", "site-ready") - if err = <-control.Done(); err != nil { - return fmt.Errorf("Piik App native control stopped unexpectedly: %w", err) - } - return nil } func runLocal(ctx context.Context, options Options, config appconfig.Config, @@ -533,7 +581,8 @@ func runLocal(ctx context.Context, options Options, config appconfig.Config, options.console.show(view) if !options.DisableBrowser { options.console.openBrowser(launchURL) - } else if options.Ready != nil { + } + if options.Ready != nil { options.Ready(launchURL) } var tunnelDone <-chan struct{} diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 88f90a8ec..2477c849a 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "log" "log/slog" "net" @@ -22,7 +23,6 @@ import ( tea "charm.land/bubbletea/v2" appconfig "github.com/TNTcraftHIM/Piik/internal/app/config" "github.com/TNTcraftHIM/Piik/internal/app/lan" - "github.com/TNTcraftHIM/Piik/internal/app/launcher" "github.com/TNTcraftHIM/Piik/internal/app/loopback" serverconfig "github.com/TNTcraftHIM/Piik/internal/server/config" ) @@ -226,7 +226,7 @@ func TestLauncherRetainsItsRuntimeFailureDuringCancellation(t *testing.T) { go func() { defer close(requestDone) client := &http.Client{Timeout: 3 * time.Second} - response, err := client.Post(strings.TrimSuffix(target, "/client")+"/api/client-launcher/launch", + response, err := postLocalLauncher(client, strings.TrimSuffix(target, "/client")+"/api/client-launcher/launch", "application/json", strings.NewReader(`{"mode":"link","language":"en"}`)) if err != nil { if !cancelBeforeFailure { @@ -304,7 +304,9 @@ func TestHungBrowserHandoffDoesNotBlockLauncherOrSiteExit(t *testing.T) { done := make(chan error, 1) go func() { if mode == "site" { - done <- runSite("https://share.example", options, control) + options.SiteSet = true + config.Site = "https://share.example" + done <- runLauncher(ctx, options, configPath, config, control, func(*Options, appconfig.Config) error { return nil }) } else { done <- runLauncher(ctx, options, configPath, config, control, func(*Options, appconfig.Config) error { return nil }) } @@ -369,7 +371,9 @@ func TestBrowserOpenFailureKeepsTheAppAvailable(t *testing.T) { done := make(chan error, 1) go func() { if mode == "site" { - done <- runSite("https://share.example", options, control) + options.SiteSet = true + config.Site = "https://share.example" + done <- runLauncher(ctx, options, configPath, config, control, func(*Options, appconfig.Config) error { return nil }) } else { done <- runLauncher(ctx, options, configPath, config, control, func(*Options, appconfig.Config) error { return nil }) @@ -404,20 +408,22 @@ func TestBrowserOpenFailureKeepsTheAppAvailable(t *testing.T) { if mode == "launcher" { client := &http.Client{Timeout: 3 * time.Second} origin := strings.TrimSuffix(view.entry, "/client") - response, err := client.Post(origin+"/api/client-launcher/launch", "application/json", - strings.NewReader(`{"mode":"site","language":"en","site":"https://share.example"}`)) + + response, err := postLocalLauncher(client, origin+"/api/client-launcher/site", "application/json", + strings.NewReader(`{"site":"https://share.example","expectedSite":""}`)) if err != nil { t.Fatal(err) } defer response.Body.Close() - var result struct{ Target string } - if json.NewDecoder(response.Body).Decode(&result) != nil || response.StatusCode != http.StatusOK || - result.Target != launchURL("https://share.example") { - t.Fatalf("manual launch failed: status=%d, target=%q", response.StatusCode, result.Target) + var result struct{ Site string } + if json.NewDecoder(response.Body).Decode(&result) != nil || response.StatusCode != http.StatusOK || result.Site != "https://share.example" { + t.Fatalf("manual approval failed: status=%d, site=%q", response.StatusCode, result.Site) } - if saved, err := launcher.LoadMode(configPath); err != nil || saved != launcher.ModeSite { - t.Fatalf("manual selection was not saved: %q, %v", saved, err) + saved, err := appconfig.LoadOrCreate(configPath) + if err != nil || saved.Site != result.Site { + t.Fatalf("approval was not saved: %v", err) } + } }) } @@ -743,3 +749,14 @@ func TestLocalServerOwnsItsSTUNConfiguration(t *testing.T) { }) } } + +func postLocalLauncher(client *http.Client, target, contentType string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPost, target, body) + if err != nil { + return nil, err + } + address, _ := url.Parse(target) + req.Header.Set("Origin", address.Scheme+"://"+address.Host) + req.Header.Set("Content-Type", contentType) + return client.Do(req) +} diff --git a/internal/app/console.go b/internal/app/console.go index 0a4bb04da..bc3998a5f 100644 --- a/internal/app/console.go +++ b/internal/app/console.go @@ -87,7 +87,7 @@ var consoleCopy = map[string][3]string{ "local": {"Local network", "局域网", "□"}, "link": {"Public link", "公网链接", "↗"}, "site": {"Piik Site", "Piik 站点", "@"}, - "setup": {"Choose a mode in the browser", "在浏览器中选择模式", "?"}, + "setup": {"App is ready", "App 已就绪", "?"}, "starting": {"Starting", "正在启动", "…"}, "ready": {"Ready", "已就绪", "✓"}, "stopping": {"Stopping", "正在退出", "→"}, diff --git a/internal/app/launcher/preference.go b/internal/app/launcher/preference.go index facb77351..cf28634e8 100644 --- a/internal/app/launcher/preference.go +++ b/internal/app/launcher/preference.go @@ -31,5 +31,5 @@ func SaveMode(configPath string, mode Mode) error { } func validMode(mode Mode) bool { - return mode == ModeLocal || mode == ModeLink || mode == ModeSite + return mode == ModeLocal || mode == ModeLink } diff --git a/internal/app/launcher/preference_test.go b/internal/app/launcher/preference_test.go index b478395b7..60d4ff305 100644 --- a/internal/app/launcher/preference_test.go +++ b/internal/app/launcher/preference_test.go @@ -27,10 +27,8 @@ func TestSavedModeSurvivesRelaunchWithoutChangingLegacyConfig(t *testing.T) { site string want Mode }{ - {ModeSite, config.Site, ModeSite}, {ModeLocal, config.Site, ModeLocal}, {ModeLink, config.Site, ModeLink}, - {ModeSite, "", ModeLink}, } { if err := SaveMode(path, test.chosen); err != nil { t.Fatal(err) diff --git a/internal/app/launcher/server.go b/internal/app/launcher/server.go index a63b0de45..9763bb26f 100644 --- a/internal/app/launcher/server.go +++ b/internal/app/launcher/server.go @@ -8,6 +8,7 @@ import ( "io" "io/fs" "log/slog" + "mime" "net" "net/http" "net/url" @@ -34,13 +35,11 @@ type Mode string const ( ModeLocal Mode = "local" ModeLink Mode = "link" - ModeSite Mode = "site" ) type Selection struct { Mode Mode `json:"mode"` Language string `json:"language"` - Site string `json:"site,omitempty"` LocalAccessPassword string `json:"localAccessPassword"` Debug bool `json:"debug,omitempty"` LANAddress *string `json:"lanAddress,omitempty"` @@ -54,6 +53,8 @@ type Options struct { LocalAccessPassword string Debug bool LANAddress string + ChangeSite func(string) (string, error) + EnableDebug func() error } var listLANAddresses = lan.List @@ -72,6 +73,8 @@ type Server struct { revision string debug bool lanAddress string + changeSite func(string) (string, error) + enableDebug func() error selection chan Selection resultReady chan struct{} handled chan struct{} @@ -123,6 +126,8 @@ func Start(parent context.Context, assets fs.FS, options Options) (*Server, erro revision: strings.TrimSpace(options.Revision), debug: options.Debug, lanAddress: options.LANAddress, + changeSite: options.ChangeSite, + enableDebug: options.EnableDebug, selection: make(chan Selection, 1), resultReady: make(chan struct{}), handled: make(chan struct{}), @@ -159,6 +164,16 @@ func (server *Server) Done() <-chan error { return server.done } +func (server *Server) Begin() bool { + server.mu.Lock() + defer server.mu.Unlock() + if server.selected { + return false + } + server.selected = true + return true +} + func (server *Server) SetResult(target string, resultErr error) { server.resultOnce.Do(func() { if resultErr == nil && !validTarget(target) { @@ -196,11 +211,18 @@ func (server *Server) ServeHTTP(response http.ResponseWriter, request *http.Requ return } response.Header().Set("X-Content-Type-Options", "nosniff") + response.Header().Set("Content-Security-Policy", "frame-ancestors 'none'") + response.Header().Set("X-Frame-Options", "DENY") + response.Header().Set("Referrer-Policy", "no-referrer") switch request.URL.Path { case "/api/client-launcher": server.handleState(response, request) case "/api/client-launcher/launch": server.handleLaunch(response, request) + case "/api/client-launcher/site": + server.handleSite(response, request) + case "/api/client-launcher/debug": + server.handleDebug(response, request) case "/client", "/client/": if request.Method != http.MethodGet && request.Method != http.MethodHead { response.Header().Set("Allow", "GET, HEAD") @@ -232,24 +254,97 @@ func (server *Server) handleState(response http.ResponseWriter, request *http.Re values = append(values, address.Address) } selected, _ := lan.Select(values, server.lanAddress) - writeJSON(response, http.StatusOK, map[string]any{ + server.mu.Lock() + state := map[string]any{ "site": server.site, "localAccessPassword": server.localAccessPassword, "version": server.version, "revision": server.revision, "packageTarget": runtime.GOOS + "-" + runtime.GOARCH, "debug": server.debug, + "target": server.target, + "starting": server.selected && !server.resultSet, + "detail": errorDetail(server.resultErr), "lan": map[string]any{"addresses": addresses, "selected": selected}, "defaultMode": func() Mode { - if validMode(server.defaultMode) && (server.defaultMode != ModeSite || server.site != "") { + if server.defaultMode == ModeLocal || server.defaultMode == ModeLink { return server.defaultMode } - if server.site != "" { - return ModeSite - } return ModeLink }(), - }) + } + server.mu.Unlock() + writeJSON(response, http.StatusOK, state) +} + +func (server *Server) handleSite(response http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost { + response.Header().Set("Allow", http.MethodPost) + http.Error(response, "method not allowed", http.StatusMethodNotAllowed) + return + } + request.Body = http.MaxBytesReader(response, request.Body, maxRequestBytes) + var change struct { + Site string `json:"site"` + ExpectedSite string `json:"expectedSite"` + } + decoder := json.NewDecoder(request.Body) + decoder.DisallowUnknownFields() + if decoder.Decode(&change) != nil || decoder.Decode(&struct{}{}) != io.EOF { + http.Error(response, "invalid site request", http.StatusBadRequest) + return + } + site, err := appconfig.NormalizeSite(change.Site) + if err != nil { + http.Error(response, "invalid site origin", http.StatusBadRequest) + return + } + server.mu.Lock() + if server.site != change.ExpectedSite { + current := server.site + server.mu.Unlock() + writeJSON(response, http.StatusConflict, map[string]string{"site": current}) + return + } + if server.changeSite == nil { + server.mu.Unlock() + http.Error(response, "site approval is unavailable", http.StatusServiceUnavailable) + return + } + current, err := server.changeSite(site) + // Revocation can succeed while media cleanup times out. Reflect the actual + // saved permission even on error; a stale confirmation must not restore it. + server.site = current + server.mu.Unlock() + if err != nil { + writeJSON(response, http.StatusServiceUnavailable, map[string]string{"site": current, "detail": errorDetail(err)}) + return + } + writeJSON(response, http.StatusOK, map[string]string{"site": current}) +} + +func (server *Server) handleDebug(response http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost { + response.Header().Set("Allow", http.MethodPost) + http.Error(response, "method not allowed", http.StatusMethodNotAllowed) + return + } + server.mu.Lock() + if !server.debug { + if server.selected || server.enableDebug == nil { + server.mu.Unlock() + http.Error(response, "enable diagnostics before starting a local room", http.StatusConflict) + return + } + if err := server.enableDebug(); err != nil { + server.mu.Unlock() + writeJSON(response, http.StatusServiceUnavailable, map[string]string{"detail": errorDetail(err)}) + return + } + server.debug = true + } + server.mu.Unlock() + writeJSON(response, http.StatusOK, map[string]bool{"debug": true}) } func (server *Server) handleLaunch(response http.ResponseWriter, request *http.Request) { @@ -266,16 +361,12 @@ func (server *Server) handleLaunch(response http.ResponseWriter, request *http.R }) return } - server.mu.Lock() - if server.selected { - server.mu.Unlock() + if !server.Begin() { writeJSON(response, http.StatusConflict, map[string]string{ "error": "launch already requested", "detail": "Close and reopen Piik before starting again.", }) return } - server.selected = true - server.mu.Unlock() select { case server.selection <- selection: @@ -339,6 +430,10 @@ func (server *Server) requestAllowed(request *http.Request) bool { return false } origin := strings.TrimSpace(request.Header.Get("Origin")) + if request.Method == http.MethodPost { + mediaType, _, err := mime.ParseMediaType(request.Header.Get("Content-Type")) + return err == nil && mediaType == "application/json" && sameOrigin(origin, "http://"+request.Host) + } return origin == "" || sameOrigin(origin, "http://"+request.Host) } @@ -373,19 +468,6 @@ func decodeSelection(reader io.Reader) (Selection, error) { } switch selection.Mode { case ModeLocal, ModeLink: - if strings.TrimSpace(selection.Site) != "" { - return Selection{}, errors.New("local launch cannot include a Site") - } - selection.Site = "" - case ModeSite: - if selection.LocalAccessPassword != "" { - return Selection{}, errors.New("Site launch cannot include a local access password") - } - normalized, err := appconfig.NormalizeSite(selection.Site) - if err != nil || normalized == "" { - return Selection{}, errors.New("Site launch requires an HTTP or HTTPS origin") - } - selection.Site = normalized default: return Selection{}, errors.New("unknown launch mode") } diff --git a/internal/app/launcher/server_test.go b/internal/app/launcher/server_test.go index 5ceeb5c75..0f7b657e4 100644 --- a/internal/app/launcher/server_test.go +++ b/internal/app/launcher/server_test.go @@ -4,8 +4,10 @@ import ( "bytes" "encoding/json" "errors" + "io" "io/fs" "net/http" + "net/url" "runtime" "strings" "testing" @@ -40,7 +42,7 @@ func TestLauncherServesStateAndCompletesOneSelection(t *testing.T) { Debug *bool `json:"debug"` } if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&state) != nil || - state.Site != "https://share.example" || state.DefaultMode != ModeSite || + state.Site != "https://share.example" || state.DefaultMode != ModeLink || state.Version != "development" || state.Revision != "" || state.LocalAccessPassword != "" || state.Debug == nil || *state.Debug { t.Fatalf("launcher state = %d, %+v", response.StatusCode, state) } @@ -48,7 +50,7 @@ func TestLauncherServesStateAndCompletesOneSelection(t *testing.T) { result := make(chan *http.Response, 1) requestErr := make(chan error, 1) go func() { - response, err := http.Post( + response, err := localPost( origin+"/api/client-launcher/launch", "application/json", bytes.NewBufferString(`{"mode":"link","language":"vis","debug":true}`), @@ -148,7 +150,7 @@ func TestLauncherLANChoicesUseCurrentAddressesAndTheExplicitPreference(t *testin } } if err := json.NewDecoder(response.Body).Decode(&state); err != nil || response.StatusCode != http.StatusOK || - state.DefaultMode != ModeSite || state.LAN == nil || state.LAN.Addresses == nil || + state.DefaultMode != ModeLink || state.LAN == nil || state.LAN.Addresses == nil || state.LAN.Selected != test.selected || len(state.LAN.Addresses) != len(test.addresses) { t.Fatalf("LAN choices = %+v, status %d, %v", state, response.StatusCode, err) } @@ -164,7 +166,7 @@ func TestLauncherLANChoicesUseCurrentAddressesAndTheExplicitPreference(t *testin func TestLauncherPassesTheChosenLANAddressToTheApp(t *testing.T) { server := startFixture(t, "") server.SetResult("http://localhost:8787/#piik-client=1", nil) - response, err := http.Post(strings.TrimSuffix(server.URL(), "/client")+"/api/client-launcher/launch", + response, err := localPost(strings.TrimSuffix(server.URL(), "/client")+"/api/client-launcher/launch", "application/json", strings.NewReader(`{"mode":"local","language":"en","lanAddress":"192.168.1.4"}`)) if err != nil { t.Fatal(err) @@ -206,7 +208,7 @@ func TestLauncherPreservesAnOptionalLocalPassword(t *testing.T) { t.Fatal(err) } go func() { - response, err := http.Post( + response, err := localPost( origin+"/api/client-launcher/launch", "application/json", bytes.NewReader(payload), @@ -251,25 +253,6 @@ func TestLauncherRequiresTheEmbeddedBrowserBuild(t *testing.T) { } } -func TestLauncherNormalizesSiteSelection(t *testing.T) { - for input, want := range map[string]string{ - " Share.Example:443/ ": "https://share.example", - "https://Share.Example/": "https://share.example", - "http://192.168.1.4:8787/": "http://192.168.1.4:8787", - } { - t.Run(input, func(t *testing.T) { - payload, err := json.Marshal(Selection{Mode: ModeSite, Language: "en", Site: input}) - if err != nil { - t.Fatal(err) - } - selection, err := decodeSelection(bytes.NewReader(payload)) - if err != nil || selection.Site != want { - t.Fatalf("site selection = %+v, %v; want %q", selection, err, want) - } - }) - } -} - func TestLauncherRejectsInvalidSelections(t *testing.T) { for name, payload := range map[string]string{ "unknown": `{"mode":"other","language":"en"}`, @@ -294,7 +277,7 @@ func TestLauncherRejectsInvalidSelections(t *testing.T) { t.Run(name, func(t *testing.T) { server := startFixture(t, "") origin := strings.TrimSuffix(server.URL(), "/client") - response, err := http.Post( + response, err := localPost( origin+"/api/client-launcher/launch", "application/json", bytes.NewBufferString(payload), @@ -331,7 +314,7 @@ func TestLauncherReportsSafeBoundedFailureDetails(t *testing.T) { } _ = response.Body.Close() server.SetResult("", errors.New("public tunnel failed: token=private-secret "+strings.Repeat("network unavailable ", 200))) - response, err = http.Post(origin+"/api/client-launcher/launch", "application/json", + response, err = localPost(origin+"/api/client-launcher/launch", "application/json", strings.NewReader(`{"mode":"link","language":"en"}`)) if err != nil { t.Fatal(err) @@ -411,3 +394,14 @@ func startFixtureWithBuildAndPassword( t.Cleanup(func() { _ = server.Close() }) return server } + +func localPost(target, contentType string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPost, target, body) + if err != nil { + return nil, err + } + address, _ := url.Parse(target) + req.Header.Set("Origin", address.Scheme+"://"+address.Host) + req.Header.Set("Content-Type", contentType) + return (&http.Client{Timeout: 3 * time.Second}).Do(req) +} diff --git a/internal/app/launcher/site_access_test.go b/internal/app/launcher/site_access_test.go new file mode 100644 index 000000000..e8d95d837 --- /dev/null +++ b/internal/app/launcher/site_access_test.go @@ -0,0 +1,133 @@ +package launcher + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "testing" +) + +func TestSiteApprovalIsLocalExplicitAndDoesNotStartARoom(t *testing.T) { + current := "" + debugCalls := 0 + server, err := Start(t.Context(), appAssets(), Options{ + ChangeSite: func(site string) (string, error) { current = site; return current, nil }, + EnableDebug: func() error { debugCalls++; return nil }, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = server.Close() }) + origin := strings.TrimSuffix(server.URL(), "/client") + for _, path := range []string{"site", "launch", "debug"} { + for _, test := range []struct{ origin, contentType string }{ + {"", "application/json"}, {"null", "application/json"}, {"https://other.example", "application/json"}, + {origin, "text/plain"}, {origin, "application/x-www-form-urlencoded"}, + } { + request, _ := http.NewRequest("POST", origin+"/api/client-launcher/"+path, strings.NewReader(`{"site":"https://other.example","expectedSite":""}`)) + request.Header.Set("Origin", test.origin) + request.Header.Set("Content-Type", test.contentType) + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != 404 { + t.Fatalf("mutation admitted nonlocal or form request: %s %+v", path, test) + } + } + } + if current != "" || debugCalls != 0 { + t.Fatal("unapproved request had side effects") + } + for range 2 { + response, err := localPost(origin+"/api/client-launcher/debug", "application/json", strings.NewReader(`{}`)) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != 200 { + t.Fatal("standby diagnostics could not start") + } + } + if debugCalls != 1 { + t.Fatalf("diagnostic owner started %d times", debugCalls) + } + for _, test := range []struct { + site, expected, want string + status int + }{ + {"https://Share.Example:443/", "", "https://share.example", 200}, + {"https://other.example", "", "https://share.example", 409}, + {"", "https://share.example", "", 200}, + } { + body := fmt.Sprintf(`{"site":%q,"expectedSite":%q}`, test.site, test.expected) + response, err := localPost(origin+"/api/client-launcher/site", "application/json", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + var result struct{ Site string } + err = json.NewDecoder(response.Body).Decode(&result) + response.Body.Close() + if err != nil || response.StatusCode != test.status || result.Site != test.want || current != test.want { + t.Fatalf("approval result: status %d, %+v, saved %q, %v", response.StatusCode, result, current, err) + } + } + select { + case <-server.Selection(): + t.Fatal("site approval created Local room authority") + default: + } + response, err := http.Get(server.URL()) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.Header.Get("X-Frame-Options") != "DENY" || response.Header.Get("Content-Security-Policy") != "frame-ancestors 'none'" { + t.Fatal("approval page can be framed") + } +} + +func TestSiteApprovalFailureReflectsTheActualPermission(t *testing.T) { + for _, applied := range []bool{false, true} { + t.Run(fmt.Sprint(applied), func(t *testing.T) { + server, err := Start(t.Context(), appAssets(), Options{Site: "https://old.example", ChangeSite: func(site string) (string, error) { + if applied { + return site, errors.New("retirement still pending") + } + return "https://old.example", errors.New("save failed") + }}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = server.Close() }) + origin := strings.TrimSuffix(server.URL(), "/client") + response, err := localPost(origin+"/api/client-launcher/site", "application/json", strings.NewReader(`{"site":"https://new.example","expectedSite":"https://old.example"}`)) + if err != nil { + t.Fatal(err) + } + var failure struct{ Site, Detail string } + err = json.NewDecoder(response.Body).Decode(&failure) + response.Body.Close() + want := "https://old.example" + if applied { + want = "https://new.example" + } + if err != nil || response.StatusCode != 503 || failure.Site != want || failure.Detail == "" { + t.Fatal("error hid actual permission") + } + response, err = http.Get(origin + "/api/client-launcher") + if err != nil { + t.Fatal(err) + } + var state struct{ Site string } + err = json.NewDecoder(response.Body).Decode(&state) + response.Body.Close() + if err != nil || state.Site != want { + t.Fatal("launcher disagrees with persisted permission") + } + }) + } +} diff --git a/internal/app/loopback/quality_preference_test.go b/internal/app/loopback/quality_preference_test.go index c5d167243..4e59b95e3 100644 --- a/internal/app/loopback/quality_preference_test.go +++ b/internal/app/loopback/quality_preference_test.go @@ -72,7 +72,7 @@ func TestQualityPreferenceRequiresCurrentOriginAndInstanceWithoutClaimingControl } server.mu.Lock() defer server.mu.Unlock() - if server.claimed != 0 { + if len(server.connections) != 0 { t.Fatal("preference claimed media control") } } diff --git a/internal/app/loopback/server.go b/internal/app/loopback/server.go index 0e56477c1..a31db0dfc 100644 --- a/internal/app/loopback/server.go +++ b/internal/app/loopback/server.go @@ -101,17 +101,27 @@ type Server struct { newControl func() ControlSession presentation func(string) qualityPreference *appconfig.QualityPreference + connectURL string done chan error mu sync.Mutex - connections map[*websocket.Conn]struct{} - claimed int + connections map[*controlConnection]struct{} closed bool controls sync.WaitGroup closeOnce sync.Once closeErr error } +// Claims start before the WebSocket upgrade so revocation also retires a +// pending handshake, even if that same origin is approved again later. +type controlConnection struct { + origin string + ctx context.Context + cancel context.CancelFunc + socket *websocket.Conn + done chan struct{} +} + func Start(parent context.Context, options Options) (*Server, error) { if parent == nil { parent = context.Background() @@ -146,7 +156,7 @@ func Start(parent context.Context, options Options) (*Server, error) { presentation: options.Presentation, qualityPreference: options.QualityPreference, done: make(chan error, 1), - connections: make(map[*websocket.Conn]struct{}), + connections: make(map[*controlConnection]struct{}), } server.httpServer = &http.Server{ Handler: server, @@ -169,9 +179,40 @@ func (server *Server) Done() <-chan error { return server.done } -func (server *Server) SetAllowedOrigins(origins []string) { +func (server *Server) SetAllowedOrigins(origins []string) error { server.mu.Lock() server.allowedOrigins = normalizedOrigins(origins) + var retired []*controlConnection + for connection := range server.connections { + if connection.ctx.Err() != nil || !server.originAllowedLocked(connection.origin) { + connection.cancel() + retired = append(retired, connection) + } + } + server.mu.Unlock() + ctx, cancel := context.WithTimeout(server.ctx, shutdownTimeout) + defer cancel() + for _, connection := range retired { + // A cancelled claim cannot attach a socket after leaving the lock. + if connection.socket != nil { + _ = connection.socket.CloseNow() + } + } + for _, connection := range retired { + select { + case <-connection.done: + case <-ctx.Done(): + // Permission already changed. Never restore a revoked origin because + // one of its media resources is slow to close. + return errors.New("App is still releasing the previous site's media; try again shortly") + } + } + return nil +} + +func (server *Server) SetConnectURL(value string) { + server.mu.Lock() + server.connectURL = value server.mu.Unlock() } @@ -181,7 +222,10 @@ func (server *Server) Close() error { server.closed = true connections := make([]*websocket.Conn, 0, len(server.connections)) for connection := range server.connections { - connections = append(connections, connection) + connection.cancel() + if connection.socket != nil { + connections = append(connections, connection.socket) + } } server.mu.Unlock() server.cancel() @@ -218,6 +262,10 @@ func (server *Server) serve() { } func (server *Server) ServeHTTP(response http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/connect-discovery" { + server.handleConnectDiscovery(response, request) + return + } if !server.requestAllowed(response, request) { return } @@ -235,6 +283,48 @@ func (server *Server) ServeHTTP(response http.ResponseWriter, request *http.Requ } } +// Discovery is the only cross-origin endpoint available before local approval. +// It exposes process identity and its local approval page, never capabilities, +// source lists, settings, or control access. A GET has no side effects. +func (server *Server) handleConnectDiscovery(response http.ResponseWriter, request *http.Request) { + if !server.hostAllowed(request.Host) { + http.NotFound(response, request) + return + } + origin := request.Header.Get("Origin") + normalized, err := appconfig.NormalizeSite(origin) + if err != nil || origin == "" || normalized != origin { + http.Error(response, "forbidden", http.StatusForbidden) + return + } + response.Header().Set("Access-Control-Allow-Origin", origin) + response.Header().Add("Vary", "Origin") + response.Header().Set("Cache-Control", "no-store") + if request.Method == http.MethodOptions { + response.Header().Set("Access-Control-Allow-Methods", http.MethodGet) + allowPrivateNetwork(response, request) + response.WriteHeader(http.StatusNoContent) + return + } + if request.Method != http.MethodGet { + response.Header().Set("Allow", "GET, OPTIONS") + http.Error(response, "method not allowed", http.StatusMethodNotAllowed) + return + } + server.mu.Lock() + connectURL := server.connectURL + server.mu.Unlock() + if connectURL == "" { + http.NotFound(response, request) + return + } + writeJSON(response, http.StatusOK, map[string]any{ + "protocol": ProtocolVersion, "service": ServiceName, + "port": server.endpoint.Port, "instanceToken": server.endpoint.InstanceToken, + "connectURL": connectURL, + }) +} + func (server *Server) handlePresentation(response http.ResponseWriter, request *http.Request) { if request.Method == http.MethodOptions { response.Header().Set("Access-Control-Allow-Methods", http.MethodPost) @@ -300,15 +390,16 @@ func (server *Server) handleControl(response http.ResponseWriter, request *http. http.Error(response, "method not allowed", http.StatusMethodNotAllowed) return } - if !server.claimConnection() { + claim := server.claimConnection(request.Header.Get("Origin")) + if claim == nil { http.Error(response, "native control capacity is occupied", http.StatusConflict) return } var connection *websocket.Conn - defer func() { server.releaseConnection(connection) }() + defer server.releaseConnection(claim) expectedSubprotocol := ControlSubprotocol + "." + server.endpoint.InstanceToken var err error - connection, err = websocket.Accept(response, request, &websocket.AcceptOptions{ + connection, err = websocket.Accept(response, request.WithContext(claim.ctx), &websocket.AcceptOptions{ Subprotocols: []string{expectedSubprotocol}, OriginPatterns: server.originPatterns(), }) @@ -320,12 +411,12 @@ func (server *Server) handleControl(response http.ResponseWriter, request *http. closeControl(connection, "unsupported control protocol") return } - if !server.setConnection(connection) { + if !server.setConnection(claim, connection) { return } connection.SetReadLimit(MaxControlMessageBytes) - helloContext, cancelHello := context.WithTimeout(server.ctx, helloTimeout) + helloContext, cancelHello := context.WithTimeout(claim.ctx, helloTimeout) messageType, payload, err := connection.Read(helloContext) cancelHello() if err != nil || messageType != websocket.MessageText { @@ -357,13 +448,12 @@ func (server *Server) handleControl(response http.ResponseWriter, request *http. } return connection.Write(ctx, websocket.MessageText, encoded) } - if err = write(server.ctx, controlMessage{ + if err = write(claim.ctx, controlMessage{ Version: ProtocolVersion, ID: message.ID, Type: "ready", }); err != nil { return } - controlContext, cancelControl := context.WithCancel(server.ctx) - defer cancelControl() + controlContext, cancelControl := claim.ctx, claim.cancel var extension ControlSession if server.newControl != nil { extension = server.newControl() @@ -403,12 +493,18 @@ func (server *Server) handleControl(response http.ResponseWriter, request *http. } }() for { + if controlContext.Err() != nil { + return + } select { case <-controlContext.Done(): return case payload = <-reads: messageType = websocket.MessageText } + if controlContext.Err() != nil { + return + } message, err = decodeEnvelope(payload) if err != nil { closeControl(connection, "invalid control message") @@ -490,6 +586,20 @@ func (server *Server) allowedOrigin(origin string) bool { return false } +// Caller holds mu. Empty Origin is reserved for per-user native tooling; +// browsers always send it when opening a cross-origin control connection. +func (server *Server) originAllowedLocked(origin string) bool { + if origin == "" || server.localOrigin(origin) { + return true + } + for _, allowed := range server.allowedOrigins { + if sameOrigin(origin, allowed) { + return true + } + } + return false +} + func (server *Server) hostAllowed(host string) bool { if host == server.endpoint.Host { return true @@ -498,40 +608,40 @@ func (server *Server) hostAllowed(host string) bool { } func (server *Server) localOrigin(origin string) bool { - parsed, err := url.Parse(origin) - return err == nil && parsed.Scheme == "http" && - parsed.Port() == strconv.Itoa(server.endpoint.Port) && - (strings.EqualFold(parsed.Hostname(), "localhost") || - strings.EqualFold(parsed.Hostname(), "127.0.0.1")) + port := strconv.Itoa(server.endpoint.Port) + return sameOrigin(origin, "http://127.0.0.1:"+port) || sameOrigin(origin, "http://localhost:"+port) } -func (server *Server) claimConnection() bool { +func (server *Server) claimConnection(origin string) *controlConnection { server.mu.Lock() defer server.mu.Unlock() - if server.closed || server.claimed >= maxControlSessions { - return false + if server.closed || len(server.connections) >= maxControlSessions || !server.originAllowedLocked(origin) { + return nil } - server.claimed++ + ctx, cancel := context.WithCancel(server.ctx) + claim := &controlConnection{origin: origin, ctx: ctx, cancel: cancel, done: make(chan struct{})} + server.connections[claim] = struct{}{} server.controls.Add(1) - return true + return claim } -func (server *Server) setConnection(connection *websocket.Conn) bool { +func (server *Server) setConnection(claim *controlConnection, connection *websocket.Conn) bool { server.mu.Lock() defer server.mu.Unlock() - if server.closed { + if server.closed || claim.ctx.Err() != nil { return false } - server.connections[connection] = struct{}{} + claim.socket = connection return true } -func (server *Server) releaseConnection(connection *websocket.Conn) { +func (server *Server) releaseConnection(connection *controlConnection) { + connection.cancel() server.mu.Lock() delete(server.connections, connection) - server.claimed-- server.mu.Unlock() server.controls.Done() + close(connection.done) } func listenInRange(bindAddress string, start, end int) (net.Listener, error) { diff --git a/internal/app/loopback/site_access_test.go b/internal/app/loopback/site_access_test.go new file mode 100644 index 000000000..3496896e4 --- /dev/null +++ b/internal/app/loopback/site_access_test.go @@ -0,0 +1,150 @@ +package loopback + +import ( + "context" + "encoding/json" + "net/http" + "testing" + "time" +) + +func TestConnectDiscoveryDoesNotGrantNativeAccess(t *testing.T) { + server := startTestServer(t, "") + server.SetConnectURL("http://127.0.0.1:18990/client") + endpoint := server.Endpoint() + for _, test := range []struct { + method, path, origin string + status int + }{ + {"GET", "/connect-discovery", testOrigin, 200}, + {"OPTIONS", "/connect-discovery", testOrigin, 204}, + {"POST", "/connect-discovery", testOrigin, 405}, + {"GET", "/connect-discovery", "null", 403}, + {"GET", "/connect-discovery", "", 403}, + {"GET", "/connect-discovery", testOrigin + "/path", 403}, + {"GET", "/health", testOrigin, 403}, + {"GET", "/control", testOrigin, 403}, + {"GET", "/quality-preference", testOrigin, 403}, + } { + request, _ := http.NewRequest(test.method, endpoint.URL+test.path, nil) + request.Header.Set("Origin", test.origin) + request.Header.Set("Access-Control-Request-Private-Network", "true") + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != test.status { + t.Fatalf("%+v: status %d", test, response.StatusCode) + } + if test.status == 200 { + var body map[string]any + if err := json.NewDecoder(response.Body).Decode(&body); err != nil { + t.Fatal(err) + } + if len(body) != 5 || body["connectURL"] != "http://127.0.0.1:18990/client" || + body["instanceToken"] != endpoint.InstanceToken || body["service"] != ServiceName { + t.Fatalf("discovery exposed more than process identity and approval address: %v", body) + } + } + response.Body.Close() + } +} + +type retiringControl struct { + *testControlSession + finish <-chan struct{} +} + +func (session *retiringControl) Close() error { + _ = session.testControlSession.Close() + <-session.finish + return nil +} + +func TestSiteChangeWaitsForMediaRetirementWithoutStoppingLocalSession(t *testing.T) { + finish := make(chan struct{}) + defer close(finish) + created := make(chan *testControlSession, 2) + server := startTestServerWithOptions(t, Options{AllowedOrigins: []string{testOrigin}, NewControl: func() ControlSession { + session := &testControlSession{events: make(chan any), closed: make(chan struct{})} + created <- session + return &retiringControl{session, finish} + }}) + endpoint := server.Endpoint() + old := dialControl(t, endpoint, endpoint.InstanceToken, testOrigin) + defer old.CloseNow() + writeControl(t, old, requestJSON("old_hello", "hello")) + var ready controlMessage + readControl(t, old, &ready) + oldSession := <-created + local := dialControl(t, endpoint, endpoint.InstanceToken, endpoint.URL) + defer local.CloseNow() + writeControl(t, local, requestJSON("local_hello", "hello")) + readControl(t, local, &ready) + localSession := <-created + + done := make(chan error, 1) + go func() { done <- server.SetAllowedOrigins([]string{"https://next.example"}) }() + select { + case <-oldSession.closed: + case <-time.After(time.Second): + t.Fatal("revoked session did not start retirement") + } + select { + case err := <-done: + t.Fatalf("permission reported ready before cleanup: %v", err) + default: + } + select { + case <-localSession.closed: + t.Fatal("unrelated Local session was closed") + default: + } + writeControl(t, local, requestJSON("local_ping", "ping")) + readControl(t, local, &ready) + if ready.Type != "pong" { + t.Fatal("Local session was interrupted") + } + // A stuck media owner must not hang consent forever or restore old permission. + select { + case err := <-done: + if err == nil { + t.Fatal("stuck cleanup was reported as successful") + } + case <-time.After(3 * time.Second): + t.Fatal("permission retirement was unbounded") + } + response := doHealthRequest(t, endpoint, endpoint.Host, testOrigin) + response.Body.Close() + if response.StatusCode != 403 { + t.Fatal("timeout restored the revoked origin") + } +} + +func TestPendingHandshakeCannotSurviveOriginReplacement(t *testing.T) { + server := startTestServer(t, testOrigin) + claim := server.claimConnection(testOrigin) + if claim == nil { + t.Fatal("initial claim failed") + } + defer server.releaseConnection(claim) + done := make(chan error, 1) + go func() { done <- server.SetAllowedOrigins(nil) }() + select { + case <-claim.ctx.Done(): + case <-time.After(time.Second): + t.Fatal("pending upgrade was not cancelled") + } + // Regrant the same origin before its old handshake returns. + server.mu.Lock() + server.allowedOrigins = []string{testOrigin} + server.mu.Unlock() + if server.setConnection(claim, nil) { + t.Fatal("old generation regained permission") + } + // Cancellation of the App also bounds an already pending permission change. + server.cancel() + if err := <-done; err == nil || claim.ctx.Err() != context.Canceled { + t.Fatal("cancelled change reported ready") + } +} diff --git a/site/film/ui/main.tsx b/site/film/ui/main.tsx index 6db1c6b6e..a4108cea1 100644 --- a/site/film/ui/main.tsx +++ b/site/film/ui/main.tsx @@ -208,7 +208,7 @@ function Screen({ shot }: { shot: Shot }) { mode={shot} onModeChange={noop} site="" - onSiteChange={noop} + onRemoveSite={noop} localAccessPassword="" onLocalAccessPasswordChange={noop} onSubmit={(event) => event.preventDefault()} diff --git a/site/index.html b/site/index.html index bcbf213ab..ec020b336 100644 --- a/site/index.html +++ b/site/index.html @@ -398,8 +398,8 @@

About this online site在线版说

The launcher opens in your system browser. Public invite creates a temporary Internet address - for friends outside your network. If you are on the same network or already have a site, see the mode descriptions.启动页会在系统浏览器里打开。「公网邀请」会生成临时公网地址,方便不在同一网络的朋友加入。如果朋友和你在同一局域网,或你们已有站点,可查看模式说明。 + for friends outside your network. Choose Local room for the same network; to use an existing site, see guide 03.启动页会在系统浏览器里打开。「公网邀请」会生成临时公网地址,方便不在同一网络的朋友加入。同一局域网可选「本地房间」;搭配已有站点使用见教程 03。

  • @@ -451,14 +451,11 @@

    Which mode should I use?模式怎 >朋友在外地、和你不在同一网络时,用临时公网邀请加入。分享期间保持 App 运行,退出后房间和邀请失效。

  • -
    -
    Connect to Site连接站点
    -
    - Connect the App to a Piik site run by you or a friend. You can also use demo.piik.tv. See guide 03 below.把 App 连接到自己或朋友部署的 Piik 站点,也可以用 demo.piik.tv。具体操作见教程 03。 -
    -
    +

    + Already have a Piik site? Keep the App running and connect from the site. + 已有 Piik 站点?保持 App 运行,从站点连接即可。 +

    Public invite relies on P2P media connections, without a media-server fallback. Some networks may @@ -516,21 +513,20 @@

    Which mode should I use?模式怎 >

    Open Piik App and choose Connect to Site. Enter the address, - add the site passphrase if required, then select Open Piik. - The sharing page opens in your browser.Keep Piik App running and open the site in your browser. Click App at the top, + check the address on the local confirmation page, then choose Allow access. + Enter any site passphrase on the site itself.打开 Piik App,选择连接站点,填入地址;如果站点有访问口令,也一并填写, - 再点击进入 Piik。浏览器会打开这个站点的分享页面。保持 Piik App 运行,在浏览器打开站点,点击顶部的 App。 + 在本机确认页核对网址,点击允许使用。如果站点要求口令,在站点页面输入。

    - After opening this site through the App once, you can keep the App running - with that site configured and visit the address directly in the same browser. - The page reconnects to the App when you choose what to share. + The same browser remembers this connection. On later visits, keep the App running; + the page reconnects when you choose what to share. Allow local-network access if your browser asks. - 通过 App 打开过一次后,保持 App 运行并使用该站点配置, - 之后就能在同一浏览器中直接访问网址,选择分享来源时会自动连接 App。 + 同一浏览器会记住这次连接。之后保持 App 运行,直接访问站点, + 选择分享来源时会自动连接 App。 浏览器询问本地网络权限时,请允许访问。

    diff --git a/src/client/App.tsx b/src/client/App.tsx index 4b7585145..75a622665 100644 --- a/src/client/App.tsx +++ b/src/client/App.tsx @@ -4,6 +4,7 @@ import { lazy, useEffect, useState, + useSyncExternalStore, type FormEvent, type ReactNode, } from "react"; @@ -20,6 +21,8 @@ import { readViewerRoute, watchViewerInvites, takeClientLaunchBootstrap, + watchClientActivation, + isClientActivated, type ViewerRoute, } from "./lib/session"; import { AppHeader } from "./components/living/Header"; @@ -99,16 +102,17 @@ function readableError(error: unknown, t: (key: "gate.connectFailed") => string) export function App() { const { lang, vis } = useCopy(); + const launchedByClient = useSyncExternalStore(watchClientActivation, isClientActivated); useEffect(installBrowserDebug, []); useEffect(() => { - if (appRoute.kind !== "client" && !clientLaunchBootstrap?.launchedByClient) return; + if (appRoute.kind !== "client" && !launchedByClient) return; const controller = new AbortController(); void import("./native/client") .then(({ notifyNativePresentation }) => notifyNativePresentation(consoleLanguage(lang, vis), controller.signal)) .catch(() => undefined); return () => controller.abort(); - }, [lang, vis]); + }, [lang, vis, launchedByClient]); return ( @@ -193,7 +197,7 @@ function AppRoute() { function RouteLoader() { return (
    - +
    @@ -202,9 +206,10 @@ function RouteLoader() { } function ViewerEntry({ route }: { route: ViewerRoute }) { + const launchedByClient = useSyncExternalStore(watchClientActivation, isClientActivated); return ( + launchedByClient={launchedByClient} /> ); } @@ -266,6 +271,7 @@ function SiteAccessGate({ viewerRoute?: ViewerRoute; }) { const { t, vis } = useCopy(); + const launchedByClient = useSyncExternalStore(watchClientActivation, isClientActivated); const [access, setAccess] = useState({ kind: "checking" }); const [capabilities, setCapabilities] = useState(null); @@ -387,7 +393,7 @@ function SiteAccessGate({ ("idle"); + const pending = useRef<{ controller: AbortController; popup: Window | null } | null>(null); + useEffect(() => () => { pending.current?.controller.abort(); pending.current?.popup?.close(); }, []); + + async function connect(): Promise { + if (pending.current) return; + // Reserve the window during the click, before local-network discovery waits. + const attempt = { controller: new AbortController(), popup: window.open("about:blank", "_blank") }; + if (!attempt.popup) { setState("unavailable"); return; } + pending.current = attempt; + setState("busy"); + const signal = attempt.controller.signal; + let native: typeof import("../../native/client") | undefined; + let approving = false; + const onMessage = async (event: MessageEvent) => { + if (approving || event.source !== attempt.popup || !approvalOrigin || event.origin !== approvalOrigin || + event.data?.type !== "piik-app-approved" || event.data.site !== window.location.origin) return; + approving = true; + // The approval message only asks us to recheck the App-owned permission. + const health = await native?.discoverNativeHealth({ signal }).catch(() => null); + if (health && !signal.aborted) activateClient(); + finish(health ? "idle" : "unavailable"); + }; + let approvalOrigin = ""; + const finish = (result: typeof state) => { + window.removeEventListener("message", onMessage); + window.removeEventListener("focus", onFocus); + attempt.controller.abort(); + if (pending.current === attempt) { pending.current = null; setState(result); } + }; + const onFocus = () => { + if (attempt.popup?.closed && !approving) finish("idle"); + }; + window.addEventListener("message", onMessage); + window.addEventListener("focus", onFocus); + signal.addEventListener("abort", () => { + window.removeEventListener("message", onMessage); + window.removeEventListener("focus", onFocus); + }, { once: true }); + try { + native = await import("../../native/client"); + if (signal.aborted) return; + const url = await native.discoverNativeConnect(signal); + if (signal.aborted) return; + if (!url) { attempt.popup.close(); finish("unavailable"); return; } + approvalOrigin = new URL(url).origin; + attempt.popup.location.replace(url); + } catch (error) { + attempt.popup.close(); + if (!signal.aborted) finish(native && error instanceof native.NativeCompatibilityError ? "incompatible" : "unavailable"); + } + } + + const label = t(state === "incompatible" ? "native.incompatible" : + state === "unavailable" ? "client.connect.unavailable" : "client.connect.action"); + return + + ; +} diff --git a/src/client/components/living/Header.tsx b/src/client/components/living/Header.tsx index 288dadef8..db91d9c7e 100644 --- a/src/client/components/living/Header.tsx +++ b/src/client/components/living/Header.tsx @@ -8,6 +8,7 @@ import { useCopy } from "../../ui/copy"; import { LanguageControl } from "./LanguageControl"; import { useTheme } from "../../ui/theme"; import { Glyph } from "../../ui/icons"; +import { AppConnectionButton } from "./AppConnectionButton"; export type LedState = "live" | "busy" | "warn" | "bad" | "off"; @@ -139,6 +140,7 @@ export function AppHeader({ {led} + {homeHref !== "/client" && } diff --git a/src/client/components/living/LauncherForm.tsx b/src/client/components/living/LauncherForm.tsx index 3df0388e5..0a6b7abec 100644 --- a/src/client/components/living/LauncherForm.tsx +++ b/src/client/components/living/LauncherForm.tsx @@ -1,4 +1,4 @@ -import { useId, type FormEvent, type ReactNode } from "react"; +import { useId, type FormEvent, type MouseEvent, type ReactNode } from "react"; import { BrandMark } from "./BrandMark"; import { Tooltip } from "./Tooltip"; import { WelcomeLine } from "./WelcomeLine"; @@ -6,7 +6,7 @@ import { Btn, Pill } from "./primitives"; import type { HintKind } from "./hints"; import { Glyph, type GlyphName } from "../../ui/icons"; import { useCopy, type CopyKey } from "../../ui/copy"; -export type AppMode = "local" | "link" | "site"; +export type AppMode = "local" | "link"; const MODES: Array<{ mode: AppMode; @@ -15,13 +15,6 @@ const MODES: Array<{ hint: CopyKey; comic: HintKind; }> = [ - { - mode: "local", - icon: "users", - label: "client.launch.local", - hint: "client.launch.localHint", - comic: "hint-client-local", - }, { mode: "link", icon: "globe", @@ -30,11 +23,11 @@ const MODES: Array<{ comic: "hint-client-link", }, { - mode: "site", - icon: "server", - label: "client.launch.site", - hint: "client.launch.siteHint", - comic: "hint-client-site", + mode: "local", + icon: "users", + label: "client.launch.local", + hint: "client.launch.localHint", + comic: "hint-client-local", }, ]; @@ -43,7 +36,11 @@ export function LauncherForm({ mode, onModeChange, site, - onSiteChange, + onRemoveSite, + siteBusy, + siteError, + target, + onOpen, localAccessPassword, onLocalAccessPasswordChange, lan, @@ -54,7 +51,11 @@ export function LauncherForm({ mode: AppMode; onModeChange: (mode: AppMode) => void; site: string; - onSiteChange: (site: string) => void; + onRemoveSite: () => void; + siteBusy?: boolean; + siteError?: string; + target?: string; + onOpen?: (event: MouseEvent) => void; localAccessPassword: string; onLocalAccessPasswordChange: (password: string) => void; lan?: { @@ -87,14 +88,33 @@ export function LauncherForm({ ); return (
    - - -

    - {t("client.launch.title")} -

    +
    + +
    +

    Piik. App

    + +
    +
    {children} +
    + +
    + {t("client.connect.standby")} +

    {t("client.connect.standbyHint")}

    + {site &&
    + {site} + +
    } + {siteError &&

    {siteError}

    } +
    +
    + + {target ? + {vis ? null : t("client.launch.go")} + : <> +

    {t("client.launch.room")}

    ) : null} - {mode === "site" ? ( - - ) : null} - - {mode !== "site" ? (
    @@ -199,7 +198,6 @@ export function LauncherForm({ {t("client.launch.localAccessHint")}

    - ) : null} choice.mode === mode)?.comic} - disabled={(mode === "site" && !site.trim()) || - (mode === "local" && lan !== undefined && !lan.selected)} + disabled={mode === "local" && lan !== undefined && !lan.selected} /> + }
    {version &&

    {t("client.launch.version", { version })} diff --git a/src/client/lib/session.ts b/src/client/lib/session.ts index e19b27cac..9eebed425 100644 --- a/src/client/lib/session.ts +++ b/src/client/lib/session.ts @@ -12,6 +12,31 @@ import { withBrowserDebug } from "./debug"; const CLIENT_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/; const CLIENT_LAUNCH_STORAGE_KEY = "piik:client-launch:v1"; +const CLIENT_ACTIVATION_EVENT = "piik-client-activation"; +let clientActivatedInPage = false; + +export function isClientActivated(): boolean { + try { return clientActivatedInPage || window.localStorage.getItem(CLIENT_LAUNCH_STORAGE_KEY) === "1"; } + catch { return clientActivatedInPage; } +} + +export function activateClient(): void { + clientActivatedInPage = true; + try { window.localStorage.setItem(CLIENT_LAUNCH_STORAGE_KEY, "1"); } catch { /* Optional persistence. */ } + window.dispatchEvent(new Event(CLIENT_ACTIVATION_EVENT)); +} + +export function watchClientActivation(changed: () => void): () => void { + const onStorage = (event: StorageEvent) => { + if (event.key === CLIENT_LAUNCH_STORAGE_KEY || event.key === null) changed(); + }; + window.addEventListener(CLIENT_ACTIVATION_EVENT, changed); + window.addEventListener("storage", onStorage); + return () => { + window.removeEventListener(CLIENT_ACTIVATION_EVENT, changed); + window.removeEventListener("storage", onStorage); + }; +} const HOST_ROOM_STORAGE_KEY = "piik:host-room:v1"; const HOST_ROOM_PREFERENCE_STORAGE_KEY = "piik:host-room-preference:v1"; const hostRoomStorageSchema = createRoomResponseSchema.pick({ @@ -139,6 +164,7 @@ export function takeClientLaunchBootstrap(): ClientLaunchBootstrap { const present = keys.some((key) => params.has(key)) || query.has("piik-preferences"); const accessValue = params.get("client-access"); const launchedFromFragment = params.get("piik-client") === "1"; + if (launchedFromFragment) clientActivatedInPage = true; const lang = params.get("piik-lang"); const mode = params.get("piik-mode"); const theme = params.get("piik-theme"); diff --git a/src/client/locales/en.ts b/src/client/locales/en.ts index 161c5fb86..758cc9e07 100644 --- a/src/client/locales/en.ts +++ b/src/client/locales/en.ts @@ -61,9 +61,22 @@ export const en: Record = { "client.launch.localHint": "Use on the same local network", "client.launch.link": "Public invite", "client.launch.linkHint": "Create one temporary public invitation link", - "client.launch.site": "Connect to Site", - "client.launch.siteHint": "Use an existing Piik Site", - "client.launch.siteAddress": "Piik Site address", + "client.launch.room": "Create a room on this device", + "client.connect.action": "Connect Piik App", + "client.connect.standby": "App is ready", + "client.connect.standbyHint": "Open a Piik Site and click App at the top to connect.", + "client.connect.remove": "Revoke this Site's App access", + "client.connect.approveTitle": "Allow this Site to use Piik App?", + "client.connect.approveHint": "This Site will be able to use this device's screens, windows and microphone. Check that you trust this address.", + "client.connect.approve": "Allow access", + "client.connect.replace": "This replaces {site} and disconnects its App sessions.", + "client.connect.approved": "This Site can now use App", + "client.connect.return": "You can close this page and return to your Site.", + "client.connect.close": "Close page", + "client.connect.changed": "The approved Site changed. Check it before trying again.", + "client.connect.saveFailed": "Could not finish the connection change. Please try again.", + "client.connect.invalid": "Invalid Site address", + "client.connect.unavailable": "Could not connect to App. Keep Piik App running and allow popups and local-network access in your browser.", "client.launch.lanAddress": "Local invitation address", "client.launch.lanHint": "Choose an address on the same network as your friends.", "client.launch.lanUnavailable": "No local network address is available. Connect to a network and refresh this page.", @@ -76,7 +89,8 @@ export const en: Record = { "client.launch.error": "Piik could not start", "client.launch.reopen": "Check the error above and the terminal, then reopen Piik. Use Debug launch to record logs.", "client.launch.debug": "Debug launch", - "client.launch.debugHint": "Record App and browser diagnostics for this launch. Press D in the terminal to export App logs.", + "client.launch.debugHint": "Record App diagnostics and logs in pages opened from here. Press D in the terminal to export App logs.", + "client.launch.debugFailed": "Could not start diagnostics. Please try again.", "client.launch.loadFailed": "Could not read launcher settings; reload to retry", "client.update.available": "A newer Piik release is available", "client.update.major": "New major Piik release", @@ -218,7 +232,7 @@ export const en: Record = { "host.sourcePicker.showCaptureBorder": "Show capture border", "host.sourcePicker.showCaptureBorderHint": "Turn off to hide the border when Windows allows it. Other captures may keep it visible.", "host.sourcePicker.empty": "No sources available; refresh the list", - "host.sourcePicker.unavailable": "Cannot connect to Piik App. Open this site from the App, refresh the list, and allow local-network access if asked.", + "host.sourcePicker.unavailable": "Cannot connect to Piik App. Keep the App running, click App at the top of this page to connect, then refresh the list.", "host.sourcePicker.unsupported": "App capture is unavailable; check the App diagnostics.", "host.sourcePicker.failed": "Could not load sources; refresh to retry.", "native.incompatible": "App and page versions are incompatible; update the App and reload.", @@ -238,7 +252,7 @@ export const en: Record = { "shareHelp.report": "If it still fails, enable diagnostics, try again and save reports from that attempt. Include the failure message, Piik version and what you were doing.", "shareHelp.appDebug": "For App logs, enable Debug launch on the mode selection page, reproduce the problem, then press D in the terminal to export. The button below records browser diagnostics only.", "shareHelp.site": "Check the site address and your access. For a self-hosted site, check PUBLIC_BASE_URL, ALLOWED_ORIGINS and the reverse proxy configuration; see the guide for steps.", - "shareHelp.appConnection": "Check that the App is running and responding. Open this site from the App, refresh the page and allow local network access if your browser asks.", + "shareHelp.appConnection": "Keep the App running and click App at the top of this page to connect. Allow local-network access if your browser asks.", "shareHelp.mediaConnection": "Check that the browser permits WebRTC and local network access, then refresh and try again. You can also compare with another browser.", "shareHelp.permission": "Return to this page, click Share and select a source again. Allow screen recording when your system or browser asks.", "shareHelp.camera": "Check camera permission and the device connection. Close applications using the camera exclusively, then select it again.", diff --git a/src/client/locales/zh.ts b/src/client/locales/zh.ts index 2e6e02930..067fda009 100644 --- a/src/client/locales/zh.ts +++ b/src/client/locales/zh.ts @@ -61,9 +61,22 @@ export const zh = { "client.launch.localHint": "同一局域网内使用", "client.launch.link": "公网邀请", "client.launch.linkHint": "生成一个临时公网邀请链接", - "client.launch.site": "连接站点", - "client.launch.siteHint": "使用已有的 Piik 站点", - "client.launch.siteAddress": "Piik 站点地址", + "client.launch.room": "在这台设备上创建房间", + "client.connect.action": "连接 Piik App", + "client.connect.standby": "App 已就绪", + "client.connect.standbyHint": "打开 Piik 站点,点击顶部的 App 即可连接。", + "client.connect.remove": "取消此站点的 App 授权", + "client.connect.approveTitle": "允许此站点使用 Piik App?", + "client.connect.approveHint": "此站点将可以使用本机的屏幕、窗口和麦克风。请确认你信任这个网址。", + "client.connect.approve": "允许使用", + "client.connect.replace": "将替换 {site},并断开它的 App 连接。", + "client.connect.approved": "已允许此站点使用 App", + "client.connect.return": "可以关闭此页面,返回原站点继续。", + "client.connect.close": "关闭此页", + "client.connect.changed": "授权站点已改变,请核对当前站点后重试。", + "client.connect.saveFailed": "未能完成连接更改,请重试。", + "client.connect.invalid": "站点网址无效", + "client.connect.unavailable": "未能连接 App。请保持 Piik App 运行,并允许浏览器打开新窗口和访问本地网络。", "client.launch.lanAddress": "局域网邀请地址", "client.launch.lanHint": "选择与朋友处于同一网络的地址。", "client.launch.lanUnavailable": "没有可用的局域网地址,请连接网络后刷新页面", @@ -76,7 +89,8 @@ export const zh = { "client.launch.error": "Piik 启动失败", "client.launch.reopen": "查看上方原因和终端详情,重新打开 Piik 后重试。可开启「诊断启动」记录日志。", "client.launch.debug": "诊断启动", - "client.launch.debugHint": "本次启动记录 App 和网页诊断。App 日志可在终端按 D 导出。", + "client.launch.debugHint": "开启 App 诊断,从此页进入的网页也会记录日志。App 日志可在终端按 D 导出。", + "client.launch.debugFailed": "未能开启诊断,请重试。", "client.launch.loadFailed": "无法读取启动设置,请刷新页面重试", "client.update.available": "发现新的 Piik 版本", "client.update.major": "发现新的 Piik 大版本", @@ -218,7 +232,7 @@ export const zh = { "host.sourcePicker.showCaptureBorder": "显示采集边框", "host.sourcePicker.showCaptureBorderHint": "关闭后尝试隐藏黄框;Windows 或其他采集程序可能仍要求显示。", "host.sourcePicker.empty": "暂无可用来源,请刷新列表", - "host.sourcePicker.unavailable": "未连接 Piik App。请从 App 打开本站后刷新列表,并允许浏览器的本地网络访问。", + "host.sourcePicker.unavailable": "未连接 Piik App。请保持 App 运行,点击页面顶部的 App 完成连接,再刷新列表。", "host.sourcePicker.unsupported": "App 采集暂不可用,请查看 App 诊断信息。", "host.sourcePicker.failed": "无法读取来源列表,请刷新重试。", "native.incompatible": "App 与页面不兼容,请更新 App 并刷新页面。", @@ -238,7 +252,7 @@ export const zh = { "shareHelp.report": "仍然失败时,开启诊断后再试一次,并保存同一次尝试的报告。请附上失败提示、Piik 版本和当时的操作。", "shareHelp.appDebug": "App 日志需从模式选择页开启「诊断启动」,重现后在终端按 D 导出。下方按钮只记录网页诊断。", "shareHelp.site": "确认当前站点地址和访问权限。自部署站点请核对 PUBLIC_BASE_URL、ALLOWED_ORIGINS 与反向代理配置,具体步骤见排查文档。", - "shareHelp.appConnection": "确认 App 仍在运行且没有卡住,从 App 打开当前站点后刷新页面;浏览器询问本地网络权限时,请允许访问。", + "shareHelp.appConnection": "确认 App 仍在运行,点击页面顶部的 App 完成连接;浏览器询问本地网络权限时,请允许访问。", "shareHelp.mediaConnection": "检查浏览器是否允许 WebRTC 和本地网络访问,再刷新页面重试;也可换一个浏览器对照。", "shareHelp.permission": "回到当前页面,重新点击分享并选择来源;若系统或浏览器询问录屏权限,请允许访问。", "shareHelp.camera": "检查摄像头权限,确认设备已连接且未被其他程序独占,再重新选择摄像头。", diff --git a/src/client/native/client.ts b/src/client/native/client.ts index 00ccca279..7deaf2836 100644 --- a/src/client/native/client.ts +++ b/src/client/native/client.ts @@ -18,6 +18,7 @@ import { nativeAckResponseSchema, nativeEventSchema, nativeDiscoveryIdentitySchema, + nativeConnectSchema, nativeHealthSchema, NATIVE_CLIENT_PORT_END, NATIVE_CLIENT_PORT_START, @@ -46,6 +47,7 @@ const REQUEST_TIMEOUT_MS = 8_000; interface NativeDiscoveryOptions { waitForPermission?: boolean; + signal?: AbortSignal; } async function ungrantedLocalPermission(): Promise { @@ -111,8 +113,31 @@ export class NativeRequestError extends Error { } export async function discoverNativeHealth( - { waitForPermission = true }: NativeDiscoveryOptions = {}, + options: NativeDiscoveryOptions = {}, ): Promise { + const health = await discoverNative("health", body => nativeHealthSchema.parse(body), options); + if (health) debugEvent("native", "discovered", { capabilities: health.nativeMedia }); + return health; +} + +/** A local approval address is not permission to use native media. */ +export async function discoverNativeConnect(signal?: AbortSignal): Promise { + return discoverNative("connect-discovery", body => { + const { connectURL } = nativeConnectSchema.parse(body); + const url = new URL(connectURL); + if (url.protocol !== "http:" || url.hostname !== "127.0.0.1" || !url.port || + url.username || url.password || url.pathname !== "/client" || url.search || url.hash) { + throw new Error("Invalid local App approval address"); + } + url.searchParams.set("site", window.location.origin); + return url.toString(); + }, { signal }); +} + +async function discoverNative( + path: "health" | "connect-discovery", read: (body: unknown) => T, + { waitForPermission = true, signal }: NativeDiscoveryOptions, +): Promise { if (!waitForPermission) { const permission = await ungrantedLocalPermission(); if (permission) { @@ -129,9 +154,9 @@ export async function discoverNativeHealth( length: NATIVE_CLIENT_PORT_END - NATIVE_CLIENT_PORT_START + 1, }, async (_, index) => { const port = NATIVE_CLIENT_PORT_START + index; - const response = await fetch(`http://127.0.0.1:${port}/health`, { + const response = await fetch(`http://127.0.0.1:${port}/${path}`, { cache: "no-store", - signal: controller.signal, + signal: signal ? AbortSignal.any([signal, controller.signal]) : controller.signal, targetAddressSpace: "loopback", } as RequestInit); if (!response.ok) throw new Error("Piik App discovery was not accepted"); @@ -143,9 +168,7 @@ export async function discoverNativeHealth( if (identity.data.protocol !== NATIVE_CLIENT_PROTOCOL) { throw new NativeCompatibilityError(identity.data.protocol); } - const health = nativeHealthSchema.parse(body); - debugEvent("native", "discovered", { capabilities: health.nativeMedia }); - return health; + return read(body); })); } catch (error) { const incompatible = error instanceof AggregateError diff --git a/src/client/native/wire.ts b/src/client/native/wire.ts index be0c982d9..07b97a07b 100644 --- a/src/client/native/wire.ts +++ b/src/client/native/wire.ts @@ -55,6 +55,11 @@ export const nativeHealthSchema = nativeDiscoveryIdentitySchema.extend({ }); export type NativeHealth = z.infer; +export const nativeConnectSchema = nativeDiscoveryIdentitySchema.extend({ + protocol: z.literal(NATIVE_CLIENT_PROTOCOL), + connectURL: z.string().url().max(256), +}); + const nativeWindowTargetSchema = z .object({ kind: z.literal("window"), diff --git a/src/client/pages/AppLauncherPage.tsx b/src/client/pages/AppLauncherPage.tsx index e47ebb4ab..948275d14 100644 --- a/src/client/pages/AppLauncherPage.tsx +++ b/src/client/pages/AppLauncherPage.tsx @@ -2,6 +2,7 @@ import { useEffect, useState, type FormEvent } from "react"; import { z } from "zod"; import { BrandMark } from "../components/living/BrandMark"; +import { HintComic } from "../components/living/hints"; import { LoadingStatus } from "../components/living/WaitingStatus"; import { AppReleaseNotice } from "../components/living/AppReleaseNotice"; import { AppHeader } from "../components/living/Header"; @@ -11,7 +12,6 @@ import { hasCopyPreference, useCopy } from "../ui/copy"; import { consoleLanguage } from "../locales"; import { currentThemePreference } from "../ui/theme"; import { clientLaunchURL, type ClientLaunchPresentation } from "../lib/session"; -import { browserDebugEnabled } from "../lib/debug"; import { checkReleaseUpdate, type ReleaseUpdateNotice, @@ -20,7 +20,10 @@ import { const launcherStateSchema = z.object({ site: z.string(), localAccessPassword: z.string(), - defaultMode: z.enum(["local", "link", "site"]), + defaultMode: z.enum(["local", "link"]), + target: z.string().optional(), + starting: z.boolean().optional(), + detail: z.string().optional(), revision: z.string(), version: z.string().default("development"), packageTarget: z.string().optional(), @@ -43,27 +46,57 @@ export function AppLauncherPage() { const [error, setError] = useState<{ kind: "load" | "launch"; detail?: string } | null>(null); const [appDebug, setAppDebug] = useState(); const [debug, setDebug] = useState(false); + const [debugBusy, setDebugBusy] = useState(false); + const [debugError, setDebugError] = useState(false); + const [reload, setReload] = useState(0); const [lan, setLan] = useState["lan"]>(); const [update, setUpdate] = useState(null); const [version, setVersion] = useState(); + const [target, setTarget] = useState(); + const [siteBusy, setSiteBusy] = useState(false); + const [siteError, setSiteError] = useState(); + const [approved, setApproved] = useState(false); + const requestedSite = new URLSearchParams(window.location.search).get("site"); + const approvalCurrent = approved && site === requestedSite; + const validSite = requestedSite && (() => { + try { + const url = new URL(requestedSite); + return ["http:", "https:"].includes(url.protocol) && url.origin === requestedSite; + } catch { return false; } + })(); useEffect(() => { let current = true; - void fetch("/api/client-launcher", { cache: "no-store" }) + let timer: ReturnType | undefined; + let checkedRelease = false; + let reading = false; + const readState = (restoreSettings = false) => { + if (reading || !current) return; + reading = true; + clearTimeout(timer); + void fetch("/api/client-launcher", { cache: "no-store" }) .then(async (response) => { if (!response.ok) throw new Error(); return launcherStateSchema.parse(await response.json()); }) .then((state) => { if (!current) return; - setMode(state.defaultMode); + if (restoreSettings) { + setMode(state.defaultMode); + setLocalAccessPassword(state.localAccessPassword); + setLan(state.lan); + } setVersion(state.version); + setTarget(state.target); + setStarting(state.starting === true); setSite(state.site); - setLocalAccessPassword(state.localAccessPassword); setAppDebug(state.debug); - setDebug(state.debug === true || browserDebugEnabled); - setLan(state.lan); + setDebug(state.debug === true); + if (state.detail) setError({ kind: "launch", detail: state.detail }); setLoading(false); + if (state.starting) timer = setTimeout(() => void readState(), 750); + if (checkedRelease) return; + checkedRelease = true; void checkReleaseUpdate({ version: state.version, revision: state.revision, @@ -76,19 +109,36 @@ export function AppLauncherPage() { .catch(() => { if (!current) return; setError({ kind: "load" }); + setStarting(false); setLoading(false); - }); + }).finally(() => { reading = false; }); + }; + void readState(true); + const refresh = () => { void readState(); }; + window.addEventListener("focus", refresh); return () => { current = false; + clearTimeout(timer); + window.removeEventListener("focus", refresh); }; - }, []); + }, [reload]); - async function launch(event: FormEvent): Promise { - event.preventDefault(); - if (starting || (mode === "site" && !site.trim()) || - (mode === "local" && lan !== undefined && !lan.selected)) return; - setStarting(true); - setError(null); + async function enableDebug(): Promise { + if (debugBusy) return; + setDebugBusy(true); + setDebugError(false); + try { + const response = await fetch("/api/client-launcher/debug", { + method: "POST", headers: { "Content-Type": "application/json" }, body: "{}", + }); + if (!response.ok) throw new Error(); + setAppDebug(true); + setDebug(true); + } catch { setDebugError(true); } + finally { setDebugBusy(false); } + } + + function launchURL(target: string): string { const theme = currentThemePreference(); const presentation: ClientLaunchPresentation = { lang, vis, theme, @@ -97,6 +147,15 @@ export function AppLauncherPage() { ...(theme !== null ? ["theme" as const] : []), ], }; + return clientLaunchURL(target, presentation, debug); + } + + async function launch(event: FormEvent): Promise { + event.preventDefault(); + if (starting || + (mode === "local" && lan !== undefined && !lan.selected)) return; + setStarting(true); + setError(null); try { const response = await fetch("/api/client-launcher/launch", { method: "POST", @@ -106,9 +165,14 @@ export function AppLauncherPage() { language: consoleLanguage(lang, vis), ...(appDebug !== undefined && debug ? { debug: true } : {}), ...(mode === "local" && lan ? { lanAddress: lan.selected } : {}), - ...(mode === "site" ? { site } : { localAccessPassword }), + localAccessPassword, }), }); + if (response.status === 409) { + setLoading(true); + setReload(value => value + 1); + return; + } if (!response.ok) { const failure = launcherErrorSchema.safeParse(await response.json().catch(() => null)); setError({ kind: "launch", detail: failure.success ? failure.data.detail : undefined }); @@ -116,23 +180,50 @@ export function AppLauncherPage() { return; } const result = launcherResultSchema.parse(await response.json()); - window.location.replace(clientLaunchURL(result.target, presentation, debug)); + window.location.replace(launchURL(result.target)); } catch { setError({ kind: "launch" }); setStarting(false); } } + async function changeSite(nextSite: string): Promise { + if (siteBusy) return; + setSiteBusy(true); + setSiteError(undefined); + try { + const response = await fetch("/api/client-launcher/site", { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ site: nextSite, expectedSite: site }), + }); + const result = z.object({ site: z.string() }).safeParse(await response.json().catch(() => null)); + if (result.success) setSite(result.data.site); + if (response.status === 409 && result.success) { + setSiteError(t("client.connect.changed")); + return; + } + if (!response.ok || !result.success) throw new Error("App approval failed"); + setSite(result.data.site); + if (requestedSite && result.data.site === requestedSite) { + setApproved(true); + window.opener?.postMessage({ type: "piik-app-approved", site: requestedSite }, requestedSite); + } + } catch { + setSiteError(t("client.connect.saveFailed")); + } finally { setSiteBusy(false); } + } + return (

    setDebug((value) => !value)} /> + disabled={appDebug || debugBusy || loading || starting || !!target || error !== null} + onClick={() => void enableDebug()} /> )} />
    - {loading || starting ? ( + {debugError && } + {loading || (starting && !requestedSite) ? ( ) : error ? (
    @@ -155,6 +246,23 @@ export function AppLauncherPage() { onClick={() => window.location.reload()} />}
    + ) : requestedSite ? ( +
    + + {vis && } +

    {t(approvalCurrent ? "client.connect.approved" : "client.connect.approveTitle")}

    +
    {requestedSite}
    +

    {t(approvalCurrent ? "client.connect.return" : "client.connect.approveHint")}

    + {!approvalCurrent && site && site !== requestedSite &&

    {t("client.connect.replace", { site })}

    } + {!validSite && } + {siteError &&

    {siteError}

    } +
    + window.close()} /> + {!approvalCurrent && void changeSite(requestedSite)} />} +
    +
    ) : ( void changeSite("")} + siteBusy={siteBusy} + siteError={siteError} + target={target} + onOpen={(event) => { event.currentTarget.href = launchURL(event.currentTarget.href); }} onLocalAccessPasswordChange={setLocalAccessPassword} lan={lan && { ...lan, onChange: (selected) => setLan({ ...lan, selected }) }} onSubmit={launch} diff --git a/src/client/pages/ControlsPreview.tsx b/src/client/pages/ControlsPreview.tsx index a603c0286..1d624a3ed 100644 --- a/src/client/pages/ControlsPreview.tsx +++ b/src/client/pages/ControlsPreview.tsx @@ -250,8 +250,8 @@ export function ControlsPreview() {
    - setLaunchSite("")} lan={{ selected: launchLan, onChange: setLaunchLan, addresses: [ { address: "192.0.2.10", name: "Wi-Fi" }, { address: "192.0.2.20", name: "vEthernet (Example Virtual Network Adapter with a Long Name)" }, diff --git a/src/client/pages/ViewerPage.tsx b/src/client/pages/ViewerPage.tsx index 1339113da..34f04aab0 100644 --- a/src/client/pages/ViewerPage.tsx +++ b/src/client/pages/ViewerPage.tsx @@ -171,6 +171,8 @@ export function ViewerPage({ launchedByClient = false, }: ViewerPageProps) { const { lang, t, vis, titleFrames } = useCopy(); + const clientActivationRef = useRef(launchedByClient); + clientActivationRef.current = launchedByClient; const viewerClientId = useMemo( () => getStableClientId("viewer", roomId), [roomId], @@ -489,7 +491,7 @@ export function ViewerPage({ }; const acquireNativeClient = async (): Promise => { - if (!active || !launchedByClient || !nativeViewerAvailable) return null; + if (!active || !clientActivationRef.current || !nativeViewerAvailable) return null; if (!nativeClientPromise) { const connecting: Promise = NativeClient.connect({ waitForPermission: false }).catch(() => null).then((client) => { if (!client || !active || nativeClientPromise !== connecting) { @@ -514,7 +516,7 @@ export function ViewerPage({ events: ViewerPeerEvents, options: ViewerPeerOptions, ): ViewerMediaPeer { - if (!launchedByClient) { + if (!clientActivationRef.current) { return new ViewerPeer(iceConfig, events, options); } return new NativeCapableViewerPeer( @@ -1841,7 +1843,7 @@ export function ViewerPage({ viewerRelaySourceKey = null; releaseNativeClient(); }; - }, [roomId, viewerGrant, viewerPasswordAttempt, launchedByClient]); + }, [roomId, viewerGrant, viewerPasswordAttempt]); useEffect(() => { const video = videoRef.current; diff --git a/src/client/styles.css b/src/client/styles.css index 44e00aa6b..48fff12a2 100644 --- a/src/client/styles.css +++ b/src/client/styles.css @@ -1500,16 +1500,38 @@ html[data-theme="dark"] .lr-door-glyph { color: #7d92a8; } .lr-client-launch { flex: 1; display: grid; + grid-template-columns: minmax(0, 1fr); place-items: center; padding: 24px 16px 72px; } .lr-client-launch-panel { display: grid; - width: min(720px, 100%); + grid-template-columns: minmax(0, 1fr); + min-width: 0; + width: min(620px, 100%); justify-items: center; gap: 20px; } -.lr-client-launch-title { font-size: 24px; text-align: center; } +.lr-client-launch-heading { display: flex; align-items: center; gap: 18px; margin-bottom: 8px; width: min(380px, 100%); } +.lr-client-launch-heading > div { min-width: 0; } +.lr-client-launch-title { font-size: 36px; line-height: 1.1; letter-spacing: -.04em; } +.lr-client-launch-title small { margin-left: 6px; color: var(--muted); font-size: 15px; letter-spacing: 0; font-weight: 650; } +.lr-client-brand-dot { color: #e77d38; } +.lr-client-launch-heading .lr-welcome { text-align: left; margin-top: 8px; } +.lr-client-standby { display: flex; align-items: flex-start; gap: 14px; width: 100%; padding: 18px 20px; border: 1px solid color-mix(in srgb, var(--ink) 20%, transparent); border-radius: 22px; background: var(--paper); } +.lr-client-standby-icon { display: grid; flex: 0 0 44px; height: 44px; place-items: center; border-radius: 14px; background: var(--control-hover); color: var(--ink); } +.lr-client-standby-copy { display: grid; flex: 1; gap: 6px; min-width: 0; } +.lr-client-standby strong { font-size: 14px; } +.lr-client-standby p { margin: 0; color: var(--muted); font-size: 12px; line-height: 1.65; } +.lr-client-approved-site { display: flex; align-items: center; gap: 8px; margin-top: 4px; } +.lr-client-approved-site a { min-width: 0; overflow-wrap: anywhere; color: var(--ink); font: 12px/1.6 var(--mono); text-underline-offset: 3px; } +.lr-client-approved-site .lr-btn { min-width: 32px; width: 32px; height: 32px; padding: 0; } +.lr-client-launch-section { justify-self: start; margin: 5px 0 -8px; font-size: 13px; font-weight: 700; color: var(--muted); } +.lr-client-approval { display: grid; width: min(460px, 100%); justify-items: center; gap: 18px; text-align: center; } +.lr-client-approval h1 { font-size: 22px; line-height: 1.4; } +.lr-client-approval p { margin: 0; font-size: 13px; line-height: 1.7; color: var(--muted); } +.lr-client-approval-origin { width: 100%; padding: 18px; border: 2px solid var(--ink); border-radius: 18px; background: var(--paper); font: 15px/1.6 var(--mono); overflow-wrap: anywhere; } +.lr-client-approval-actions { display: flex; justify-content: center; flex-wrap: wrap; gap: 12px; } .lr-client-about { display: grid; gap: 4px; max-width: 100%; color: var(--muted); font-size: 12px; text-align: center; overflow-wrap: anywhere; } .lr-client-project-links { display: flex; justify-content: center; flex-wrap: wrap; gap: 0 20px; } .lr-client-project-links a { display: inline-flex; align-items: center; gap: 4px; min-height: 36px; color: inherit; text-underline-offset: 3px; } @@ -1534,7 +1556,7 @@ html[data-theme="dark"] .lr-door-glyph { color: #7d92a8; } .lr-client-modes { display: grid; width: 100%; - grid-template-columns: repeat(3, minmax(0, 1fr)); + grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px; } .lr-client-modes > .lr-comic-tip-wrap { @@ -1544,11 +1566,11 @@ html[data-theme="dark"] .lr-door-glyph { color: #7d92a8; } .lr-client-modes > .lr-comic-tip-wrap > .lr-client-mode { width: 100%; } .lr-client-mode { min-width: 0; - min-height: 112px; + min-height: 128px; place-items: center; align-content: center; gap: 10px; - padding: 14px; + padding: 18px; text-align: center; } .lr-client-mode > span { @@ -1558,7 +1580,6 @@ html[data-theme="dark"] .lr-door-glyph { color: #7d92a8; } } .lr-client-mode strong { font-size: 14px; } .lr-client-mode small { font-size: 11px; opacity: .7; } -.lr-client-site { width: min(460px, 100%); } .lr-client-access { width: min(460px, 100%); } diff --git a/tests/client-launch-presentation.test.ts b/tests/client-launch-presentation.test.ts index 7969f7136..5aa39c3bd 100644 --- a/tests/client-launch-presentation.test.ts +++ b/tests/client-launch-presentation.test.ts @@ -38,6 +38,28 @@ function browser(saved: string[] = [], blocked = false) { } describe("App presentation handoff authority", () => { + it.each([false, true])("activates the current page after approval, including blocked storage: %s", async (blocked) => { + const env = browser([], blocked); + const events = new EventTarget(); + Object.assign(env.page, { + addEventListener: events.addEventListener.bind(events), + removeEventListener: events.removeEventListener.bind(events), + dispatchEvent: events.dispatchEvent.bind(events), + }); + const { isClientActivated, activateClient, watchClientActivation } = await import("../src/client/lib/session"); + const changed = vi.fn(); + const release = watchClientActivation(changed); + expect(isClientActivated()).toBe(false); + activateClient(); + expect(isClientActivated()).toBe(true); + expect(changed).toHaveBeenCalledOnce(); + expect(env.page.location.href).toBe("https://site.example/"); + expect(env.values.get("piik:client-launch:v1")).toBe(blocked ? undefined : "1"); + release(); + activateClient(); + expect(changed).toHaveBeenCalledOnce(); + }); + it.each([ { name: "old launcher preserves saved choices", saved: ["zh", "vis", "dark"], fields: "", selection: "zh:true", theme: "dark", result: ["zh", "vis", "dark"] }, { name: "old launcher supplies only temporary first-use defaults", saved: [], fields: "", selection: "en:false", theme: "light", result: [null, null, null] }, diff --git a/tests/native-client.test.ts b/tests/native-client.test.ts index 35299606d..2818b9069 100644 --- a/tests/native-client.test.ts +++ b/tests/native-client.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { discoverNativeHealth, NativeClient, NativeCompatibilityError, notifyNativePresentation } from "../src/client/native/client"; +import { discoverNativeConnect, discoverNativeHealth, NativeClient, NativeCompatibilityError, notifyNativePresentation } from "../src/client/native/client"; import { DEFAULT_QUALITY_SETTINGS } from "../src/shared/protocol"; import { @@ -31,6 +31,45 @@ describe("native App private wire", () => { nativeMedia: { video: true, hardwareH264: true }, }; + it("discovers only a local approval page without opening native control", async () => { + vi.stubGlobal("window", { setTimeout, clearTimeout, location: new URL("https://piik.example/r/9527") }); + const socket = vi.fn(); + vi.stubGlobal("WebSocket", socket); + const fetcher = vi.fn(async (url: string) => url === `http://127.0.0.1:${health.port}/connect-discovery` + ? new Response(JSON.stringify({ ...health, connectURL: "http://127.0.0.1:54321/client" })) + : new Response(null, { status: 404 })); + vi.stubGlobal("fetch", fetcher); + await expect(discoverNativeConnect()).resolves.toBe("http://127.0.0.1:54321/client?site=https%3A%2F%2Fpiik.example"); + expect(fetcher.mock.calls.every(([url]) => url.endsWith("/connect-discovery"))).toBe(true); + expect(socket).not.toHaveBeenCalled(); + }); + + it.each([ + "https://untrusted.example/client", "http://localhost:54321/client", "http://127.0.0.1:54321/other", + "http://user@127.0.0.1:54321/client", "http://127.0.0.1:54321/client?site=https://other.example", + ])("rejects an unsafe App approval address: %s", async (connectURL) => { + vi.stubGlobal("window", { setTimeout, clearTimeout, location: new URL("https://piik.example/") }); + vi.stubGlobal("fetch", vi.fn(async (url: string) => url === `http://127.0.0.1:${health.port}/connect-discovery` + ? new Response(JSON.stringify({ ...health, connectURL })) : new Response(null, { status: 404 }))); + await expect(discoverNativeConnect()).resolves.toBeNull(); + }); + + it("cancels the whole approval scan with its caller", async () => { + vi.stubGlobal("window", { setTimeout, clearTimeout, location: new URL("https://piik.example/") }); + const signals: AbortSignal[] = []; + vi.stubGlobal("fetch", vi.fn((_url: string, init: RequestInit) => new Promise((_resolve, reject) => { + const signal = init.signal!; + signals.push(signal); + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + }))); + const controller = new AbortController(); + const result = discoverNativeConnect(controller.signal); + controller.abort(); + await expect(result).resolves.toBeNull(); + expect(signals).toHaveLength(NATIVE_CLIENT_PORT_END - NATIVE_CLIENT_PORT_START + 1); + expect(signals.every(signal => signal.aborted)).toBe(true); + }); + it("reads numeric video capability descriptions and rejects malformed limits", () => { const nativeMedia = { ...health.nativeMedia, maxVideoResolution: { width: 3840, height: 3840 }, maxVideoPixels: 3840 * 2160, maxVideoMacroblocksPerSecond: 983_040 }; diff --git a/tests/viewer-native-client-ownership.test.ts b/tests/viewer-native-client-ownership.test.ts index d16e5c7c2..22a75b5b9 100644 --- a/tests/viewer-native-client-ownership.test.ts +++ b/tests/viewer-native-client-ownership.test.ts @@ -44,7 +44,7 @@ function control() { function fixture(launchedByClient = true) { const connect = vi.fn(async (): Promise | null> => null); const context = createContext({ - launchedByClient, NativeClient: { connect }, + clientActivationRef: { current: launchedByClient }, NativeClient: { connect }, messageAuthority: { invalidate: vi.fn() }, setAssignedRoute: vi.fn(), clearViewerSfuRoute: vi.fn(), clearPeerState: vi.fn(), clearParticipantPresence: vi.fn(), dispatchPresentation: vi.fn(), From 036802201af2bc9f25a85f87cf7d3af513a3e216 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:27:54 +0800 Subject: [PATCH 165/262] feat(app): streamline site connection and launcher settings --- docs/guide/getting-started.md | 7 ++- docs/guide/getting-started.zh-CN.md | 5 ++- docs/standards/presentation-lifecycle.md | 5 +++ .../components/living/AppConnectionButton.tsx | 24 ++++++---- .../components/living/CaptureSourcePicker.tsx | 10 +++++ src/client/components/living/Header.tsx | 4 +- src/client/components/living/LauncherForm.tsx | 45 ++++++++++--------- src/client/locales/en.ts | 6 ++- src/client/locales/zh.ts | 6 ++- src/client/pages/AppLauncherPage.tsx | 13 +++--- src/client/pages/HostPage.tsx | 7 ++- src/client/styles.css | 17 +++---- tests/host-quality-ownership.test.ts | 20 ++++++++- 13 files changed, 112 insertions(+), 57 deletions(-) diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 97006eb74..f948ed11e 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -98,7 +98,7 @@ See the [App guide](../../cmd/piik-app/README.md) for package and runtime detail | Mode | When to choose it | What to keep in mind | | --- | --- | --- | -| **Local room** | Friends are on the same local network. | If several entries appear under **Local invitation address**, choose the interface and IP on your friends' network. Your computer must be reachable from theirs. A local site passphrase is optional. | +| **Local room** | Friends are on the same local network. | With multiple interfaces, expand **Local invitation address** and select the interface and IP on your friends' network. Your computer must be reachable from theirs. A local site passphrase is optional. | | **Public invite** | Friends are elsewhere and you want a temporary invitation. | Needs Internet access. The public address lasts for this App run. | For Local room and Public invite, expand **Site passphrase** to set an optional @@ -108,10 +108,13 @@ access passphrase. **App** at the top. Check the address on the local confirmation page, then choose **Allow access**. Enter any site passphrase on the site itself. Allow local-network access if the browser asks. No room mode needs to be selected in the launcher. +You can also choose **Connect Piik App** inside the **Start sharing** source +selector; its list updates as soon as the connection completes. The same browser remembers this connection. On later visits, **Start sharing** offers **Apps / Windows** and **Screens** from the running App. In another browser, -in a private window or after clearing site data, connect again. The launcher +in a private window or after clearing site data, connect again. The same exact +site already approved by App needs no repeat confirmation. The launcher shows the approved site and lets you revoke it. Approving a different site replaces the previous one and ends that site's App connections. diff --git a/docs/guide/getting-started.zh-CN.md b/docs/guide/getting-started.zh-CN.md index 540ce8e81..1d77ed917 100644 --- a/docs/guide/getting-started.zh-CN.md +++ b/docs/guide/getting-started.zh-CN.md @@ -89,7 +89,7 @@ Linux 原生采集还需要[系统组件](../../native/capture/linux/README.md) | 模式 | 什么时候用 | 需要知道 | | --- | --- | --- | -| **本地房间** | 朋友和你在同一个局域网。 | 出现多个「局域网邀请地址」时,选择与朋友同一网络的网卡和 IP。对方需要能连接到你的电脑,可以设置本地站点口令。 | +| **本地房间** | 朋友和你在同一个局域网。 | 多网卡时展开「局域网邀请地址」,选择与朋友同一网络的网卡和 IP。对方需要能连接到你的电脑,可以设置本地站点口令。 | | **公网邀请** | 朋友在别处,想临时发个邀请。 | 需要互联网连接;公网地址只在本次 App 运行期间有效。 | 本地房间和公网邀请可展开 **站点口令** 设置可选的访问口令。 @@ -97,10 +97,11 @@ Linux 原生采集还需要[系统组件](../../native/capture/linux/README.md) **已经有 Piik 站点?** 保持 App 运行,打开站点,点击顶部的 **App**。 在本机确认页核对网址,点击 **允许使用**。如果站点要求口令,在站点页面输入。 浏览器询问本地网络权限时,请允许访问。启动页无需选择房间模式。 +也可以在 **开始分享** 的选源窗口里点击 **连接 Piik App**,连接后列表会直接显示 App 来源。 同一浏览器会记住这次连接。之后直接访问站点,点击 **开始分享**, 选择 **程序 / 窗口** 或 **屏幕**,即可使用正在运行的 App。 -换浏览器、使用无痕窗口或清除站点数据后,重新连接即可。 +换浏览器、使用无痕窗口或清除站点数据后,重新连接即可;App 已允许的同一站点无需再次确认。 启动页显示已授权的站点,也可以取消授权;允许新站点会替换旧站点,并结束旧站点的 App 连接。 **网页能打开,不代表视频一定能连通。** 公网邀请为房间提供临时网页地址, diff --git a/docs/standards/presentation-lifecycle.md b/docs/standards/presentation-lifecycle.md index 72a86bd9b..3b56e312b 100644 --- a/docs/standards/presentation-lifecycle.md +++ b/docs/standards/presentation-lifecycle.md @@ -40,6 +40,11 @@ available to its approved Site before either mode starts. A Site's explicit App action discovers the local approval page, where the user confirms its exact origin. The launcher shows and revokes that approval. Activating a page supplies native capability without restarting its room or healthy media session. +The source selector exposes that same connection action before sharing; a +successful connection refreshes its current list. An already approved exact +origin needs no repeat confirmation, but the page still verifies authorized App +health. Closing the selector cancels its pending connection action. Existing +Browser shares retain their capture implementation until sharing ends. Viewer links opened at that activated origin may use the same running App without changing the Viewer UI. Starting a share offers the Browser picker and any exact App-owned native windows. It never chooses a diff --git a/src/client/components/living/AppConnectionButton.tsx b/src/client/components/living/AppConnectionButton.tsx index 6c60d63c1..c46705d39 100644 --- a/src/client/components/living/AppConnectionButton.tsx +++ b/src/client/components/living/AppConnectionButton.tsx @@ -4,7 +4,10 @@ import { useCopy } from "../../ui/copy"; import { Glyph } from "../../ui/icons"; import { Tooltip } from "./Tooltip"; -export function AppConnectionButton() { +export function AppConnectionButton({ sourcePicker = false, onConnected }: { + sourcePicker?: boolean; + onConnected?: () => void; +} = {}) { const { t, vis } = useCopy(); const [state, setState] = useState<"idle" | "busy" | "unavailable" | "incompatible">("idle"); const pending = useRef<{ controller: AbortController; popup: Window | null } | null>(null); @@ -26,14 +29,19 @@ export function AppConnectionButton() { approving = true; // The approval message only asks us to recheck the App-owned permission. const health = await native?.discoverNativeHealth({ signal }).catch(() => null); - if (health && !signal.aborted) activateClient(); + if (signal.aborted) return; finish(health ? "idle" : "unavailable"); + if (health) { + activateClient(); + onConnected?.(); + } }; let approvalOrigin = ""; const finish = (result: typeof state) => { window.removeEventListener("message", onMessage); window.removeEventListener("focus", onFocus); attempt.controller.abort(); + attempt.popup?.close(); if (pending.current === attempt) { pending.current = null; setState(result); } }; const onFocus = () => { @@ -50,24 +58,24 @@ export function AppConnectionButton() { if (signal.aborted) return; const url = await native.discoverNativeConnect(signal); if (signal.aborted) return; - if (!url) { attempt.popup.close(); finish("unavailable"); return; } + if (!url) { finish("unavailable"); return; } approvalOrigin = new URL(url).origin; attempt.popup.location.replace(url); } catch (error) { - attempt.popup.close(); if (!signal.aborted) finish(native && error instanceof native.NativeCompatibilityError ? "incompatible" : "unavailable"); } } const label = t(state === "incompatible" ? "native.incompatible" : - state === "unavailable" ? "client.connect.unavailable" : "client.connect.action"); + state === "unavailable" ? "client.connect.unavailable" : sourcePicker ? "client.connect.captureHint" : "client.connect.action"); return - ; } diff --git a/src/client/components/living/CaptureSourcePicker.tsx b/src/client/components/living/CaptureSourcePicker.tsx index ff29a7a77..7cbb748b6 100644 --- a/src/client/components/living/CaptureSourcePicker.tsx +++ b/src/client/components/living/CaptureSourcePicker.tsx @@ -10,6 +10,7 @@ import { Pill } from "./primitives"; import { CameraSources, useCameraSources } from "./CameraSources"; import { CaptureSourceCard } from "./CaptureSourceCard"; import { AudioExclusionSelect } from "./AudioExclusionSelect"; +import { AppConnectionButton } from "./AppConnectionButton"; import type { loadCameraPreviews } from "../../media/camera-previews"; const SOURCE_TABS = ["browser", "camera", "window", "display"] as const; @@ -45,6 +46,7 @@ export function CaptureSourcePicker({ onPreview, onRefresh, onCancel, + onAppConnected, browserAvailable = true, cameraAvailable = !!onCamera, initialTab = "window", @@ -70,6 +72,7 @@ export function CaptureSourcePicker({ ) => Promise; onRefresh: () => void; onCancel: () => void; + onAppConnected?: () => void; browserAvailable?: boolean; cameraAvailable?: boolean; initialTab?: SourceTab; @@ -324,6 +327,13 @@ export function CaptureSourcePicker({
    } + {onAppConnected && (nativeSources.kind === "browser" || nativeSources.kind === "unavailable") && ( +
    + + {vis ? null : {t("client.connect.captureHint")}} +
    + )} + {nativeTab && nativeSources.kind === "ready" && (supportsCaptureBorder || sources.length > 0) ? ( diff --git a/src/client/components/living/Header.tsx b/src/client/components/living/Header.tsx index db91d9c7e..67ff246f5 100644 --- a/src/client/components/living/Header.tsx +++ b/src/client/components/living/Header.tsx @@ -123,10 +123,12 @@ export function AppHeader({ led, homeHref = "/", diagnosticControl, + showAppConnection = true, }: { led?: React.ReactNode; homeHref?: string; diagnosticControl?: React.ReactNode; + showAppConnection?: boolean; }) { const { t } = useCopy(); return ( @@ -140,7 +142,7 @@ export function AppHeader({ {led} - {homeHref !== "/client" && } + {homeHref !== "/client" && showAppConnection && } diff --git a/src/client/components/living/LauncherForm.tsx b/src/client/components/living/LauncherForm.tsx index 0a6b7abec..0265d19c5 100644 --- a/src/client/components/living/LauncherForm.tsx +++ b/src/client/components/living/LauncherForm.tsx @@ -163,26 +163,7 @@ export function LauncherForm({ })}
    - {mode === "local" && lan && (lan.addresses.length !== 1 || !lan.selected) ? ( -
    - {t("client.launch.lanAddress")} - {lan.addresses.length === 0 ? ( - - ) : <> - {vis ? null :

    {t("client.launch.lanHint")}

    } -
    - {lan.addresses.map(({ address, name }) => ( - - ))} -
    - } -
    - ) : null} - +
    @@ -198,6 +179,28 @@ export function LauncherForm({ {t("client.launch.localAccessHint")}

    + {mode === "local" && lan && (lan.addresses.length !== 1 || !lan.selected) ? ( +
    + + + {t("client.launch.lanAddress")} + + + {lan.addresses.length === 0 ? ( + + ) :
    + + {t("client.launch.lanHint")} +
    } +
    + ) : null} +
    }

    {t("client.launch.credit")} - {vis && } + {vis && }

    diff --git a/src/client/locales/en.ts b/src/client/locales/en.ts index 758cc9e07..8009cf3fa 100644 --- a/src/client/locales/en.ts +++ b/src/client/locales/en.ts @@ -56,13 +56,14 @@ export const en: Record = { "client.launch.version": "Installed version {version}", "client.launch.credit": "Free and open source · TNTcraft & contributors", "client.launch.website": "Official website", - "client.launch.source": "Source on GitHub", + "client.launch.source": "GitHub", "client.launch.local": "Local room", "client.launch.localHint": "Use on the same local network", "client.launch.link": "Public invite", "client.launch.linkHint": "Create one temporary public invitation link", "client.launch.room": "Create a room on this device", "client.connect.action": "Connect Piik App", + "client.connect.captureHint": "Choose windows or screens with App.", "client.connect.standby": "App is ready", "client.connect.standbyHint": "Open a Piik Site and click App at the top to connect.", "client.connect.remove": "Revoke this Site's App access", @@ -78,6 +79,7 @@ export const en: Record = { "client.connect.invalid": "Invalid Site address", "client.connect.unavailable": "Could not connect to App. Keep Piik App running and allow popups and local-network access in your browser.", "client.launch.lanAddress": "Local invitation address", + "client.launch.chooseAddress": "Choose an address", "client.launch.lanHint": "Choose an address on the same network as your friends.", "client.launch.lanUnavailable": "No local network address is available. Connect to a network and refresh this page.", "client.launch.localAccess": "Site passphrase (optional)", @@ -232,7 +234,7 @@ export const en: Record = { "host.sourcePicker.showCaptureBorder": "Show capture border", "host.sourcePicker.showCaptureBorderHint": "Turn off to hide the border when Windows allows it. Other captures may keep it visible.", "host.sourcePicker.empty": "No sources available; refresh the list", - "host.sourcePicker.unavailable": "Cannot connect to Piik App. Keep the App running, click App at the top of this page to connect, then refresh the list.", + "host.sourcePicker.unavailable": "Cannot connect to Piik App. Keep the App running, then choose Connect Piik App.", "host.sourcePicker.unsupported": "App capture is unavailable; check the App diagnostics.", "host.sourcePicker.failed": "Could not load sources; refresh to retry.", "native.incompatible": "App and page versions are incompatible; update the App and reload.", diff --git a/src/client/locales/zh.ts b/src/client/locales/zh.ts index 067fda009..38a92aa4f 100644 --- a/src/client/locales/zh.ts +++ b/src/client/locales/zh.ts @@ -56,13 +56,14 @@ export const zh = { "client.launch.version": "当前版本 {version}", "client.launch.credit": "免费开源 · TNTcraft 与贡献者", "client.launch.website": "官方网站", - "client.launch.source": "GitHub 源码", + "client.launch.source": "GitHub", "client.launch.local": "本地房间", "client.launch.localHint": "同一局域网内使用", "client.launch.link": "公网邀请", "client.launch.linkHint": "生成一个临时公网邀请链接", "client.launch.room": "在这台设备上创建房间", "client.connect.action": "连接 Piik App", + "client.connect.captureHint": "使用 App 选择窗口或屏幕。", "client.connect.standby": "App 已就绪", "client.connect.standbyHint": "打开 Piik 站点,点击顶部的 App 即可连接。", "client.connect.remove": "取消此站点的 App 授权", @@ -78,6 +79,7 @@ export const zh = { "client.connect.invalid": "站点网址无效", "client.connect.unavailable": "未能连接 App。请保持 Piik App 运行,并允许浏览器打开新窗口和访问本地网络。", "client.launch.lanAddress": "局域网邀请地址", + "client.launch.chooseAddress": "选择地址", "client.launch.lanHint": "选择与朋友处于同一网络的地址。", "client.launch.lanUnavailable": "没有可用的局域网地址,请连接网络后刷新页面", "client.launch.localAccess": "站点口令(可选)", @@ -232,7 +234,7 @@ export const zh = { "host.sourcePicker.showCaptureBorder": "显示采集边框", "host.sourcePicker.showCaptureBorderHint": "关闭后尝试隐藏黄框;Windows 或其他采集程序可能仍要求显示。", "host.sourcePicker.empty": "暂无可用来源,请刷新列表", - "host.sourcePicker.unavailable": "未连接 Piik App。请保持 App 运行,点击页面顶部的 App 完成连接,再刷新列表。", + "host.sourcePicker.unavailable": "未连接 Piik App。请保持 App 运行,点击「连接 Piik App」。", "host.sourcePicker.unsupported": "App 采集暂不可用,请查看 App 诊断信息。", "host.sourcePicker.failed": "无法读取来源列表,请刷新重试。", "native.incompatible": "App 与页面不兼容,请更新 App 并刷新页面。", diff --git a/src/client/pages/AppLauncherPage.tsx b/src/client/pages/AppLauncherPage.tsx index 948275d14..a3cefeb35 100644 --- a/src/client/pages/AppLauncherPage.tsx +++ b/src/client/pages/AppLauncherPage.tsx @@ -55,15 +55,20 @@ export function AppLauncherPage() { const [target, setTarget] = useState(); const [siteBusy, setSiteBusy] = useState(false); const [siteError, setSiteError] = useState(); - const [approved, setApproved] = useState(false); const requestedSite = new URLSearchParams(window.location.search).get("site"); - const approvalCurrent = approved && site === requestedSite; const validSite = requestedSite && (() => { try { const url = new URL(requestedSite); return ["http:", "https:"].includes(url.protocol) && url.origin === requestedSite; } catch { return false; } })(); + const approvalCurrent = validSite && site === requestedSite; + + useEffect(() => { + // The App's saved exact origin owns permission, including repeat connections. + // The requesting page still verifies authorized health before using App. + if (approvalCurrent) window.opener?.postMessage({ type: "piik-app-approved", site: requestedSite }, requestedSite); + }, [approvalCurrent, requestedSite]); useEffect(() => { let current = true; @@ -204,10 +209,6 @@ export function AppLauncherPage() { } if (!response.ok || !result.success) throw new Error("App approval failed"); setSite(result.data.site); - if (requestedSite && result.data.site === requestedSite) { - setApproved(true); - window.opener?.postMessage({ type: "piik-app-approved", site: requestedSite }, requestedSite); - } } catch { setSiteError(t("client.connect.saveFailed")); } finally { setSiteBusy(false); } diff --git a/src/client/pages/HostPage.tsx b/src/client/pages/HostPage.tsx index d3f9d4152..c7f1978a0 100644 --- a/src/client/pages/HostPage.tsx +++ b/src/client/pages/HostPage.tsx @@ -103,6 +103,7 @@ import { deriveHostStatus, deriveParticipantStatus, type HostPhase } from "../ui import { clearHostRoom, getStableClientId, + isClientActivated, type HostRoomIdentity, type HostRoomState, mergeAuthenticatedHostRoom, @@ -1406,7 +1407,7 @@ export function HostPage({ const request = {}; nativeSourceRequestRef.current = request; nativeSourcePathRef.current = null; - if (!launchedByClient || (phase === "live" && !nativeModeRef.current)) { + if (!isClientActivated() || (phase === "live" && !nativeModeRef.current)) { setNativeSources({ kind: "browser" }); return; } @@ -3598,6 +3599,7 @@ export function HostPage({ return (
    { + if (nativeSourceRequestRef.current) void openCaptureSourcePicker(); + } : undefined} browserAvailable={!nativeActive && !!navigator.mediaDevices?.getDisplayMedia} cameraAvailable={!nativeActive && !!navigator.mediaDevices?.getUserMedia} selectionDisabled={roomMutating || switchingSource || changingQuality} diff --git a/src/client/styles.css b/src/client/styles.css index 48fff12a2..5600dd34a 100644 --- a/src/client/styles.css +++ b/src/client/styles.css @@ -544,6 +544,9 @@ body.lr-theater-open .lr-top { visibility: hidden; } scrollbar-gutter: stable; } .lr-source-picker-body > [role="status"] { text-align: center; } +.lr-source-app-connection { display: grid; justify-items: center; gap: 6px; margin-top: 12px; } +.lr-source-app-connection small { color: #9aa8bb; font-size: 12px; text-align: center; } +.lr-source-app-connect { display: flex; gap: 8px; width: auto; padding: 0 12px; margin: 0; } .lr-source-picker-list { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 240px), 1fr)); @@ -1586,17 +1589,9 @@ html[data-theme="dark"] .lr-door-glyph { color: #7d92a8; } .lr-client-access input { min-width: 0; } -.lr-client-lan { width: min(460px, 100%); min-width: 0; margin: 0; padding: 0; border: 0; } -.lr-client-lan legend { padding: 0; font-size: 14px; font-weight: 750; } -.lr-client-lan-hint { margin: 5px 0 10px; color: var(--muted); font-size: 12px; } -.lr-client-lan-options { display: grid; gap: 8px; max-height: 192px; overflow-y: auto; padding: 3px; } -.lr-client-lan-option { width: 100%; min-width: 0; height: auto; min-height: 52px; grid-template-columns: auto minmax(0, 1fr); justify-content: stretch; white-space: normal; border-radius: var(--control-radius); cursor: pointer; } -.lr-client-lan-option input { flex: 0 0 auto; width: 18px; height: 18px; margin: 0; accent-color: var(--action); } -.lr-client-lan-option span { display: grid; min-width: 0; gap: 3px; text-align: left; } -.lr-client-lan-option strong { font-size: 13px; overflow-wrap: anywhere; } -.lr-client-lan-option small { font: 12px var(--mono); } -.lr-client-lan-option:has(:focus-visible) { outline: 3px solid var(--action); outline-offset: 1px; } -.lr-client-access-options { width: min(460px, 100%); min-width: 0; } +.lr-client-room-settings { display: grid; width: min(460px, 100%); grid-template-columns: repeat(auto-fit, minmax(min(100%, 210px), 1fr)); align-items: start; gap: 12px; } +.lr-client-room-settings .lr-client-lan { padding: 8px 0 0; } +.lr-client-access-options { width: 100%; min-width: 0; } .lr-client-access-options > summary { display: flex; align-items: center; justify-content: center; gap: 8px; width: fit-content; max-width: 100%; min-height: 40px; margin: 0 auto; padding: 8px 12px; diff --git a/tests/host-quality-ownership.test.ts b/tests/host-quality-ownership.test.ts index cc8f92838..f925485b9 100644 --- a/tests/host-quality-ownership.test.ts +++ b/tests/host-quality-ownership.test.ts @@ -71,7 +71,7 @@ function fixture(launchedByClient = true) { const route = { updateProfile: vi.fn(async () => true), resyncAuthoritative: vi.fn(async (): Promise => undefined) }; const state = { debugError, debugEvent, debugOperation, NativeCompatibilityError, NativeRequestError, NativeMediaBridgeError, isCapturePermissionFailure, hostFailureCode, DOMException, - launchedByClient, hostRoomSessionAvailable: false, + launchedByClient, isClientActivated: vi.fn(() => launchedByClient), hostRoomSessionAvailable: false, NativeClient: { connect: vi.fn(async (): Promise => null) }, nativeClientConnectRef: ref | null>(null), ownNativeClient: vi.fn(), setJoiningRoom: vi.fn(), startSharing: vi.fn(), openCaptureSourcePicker: vi.fn(), @@ -941,6 +941,24 @@ describe("Host quality ownership", () => { expect(current.openCaptureSourcePicker).toHaveBeenCalledTimes(2); }); + it("refreshes a waiting picker after App approval without changing a live Browser share", async () => { + const current = fixture(false); + current.context.phase = "idle"; + await current.openPicker(); + expect(current.setNativeSources).toHaveBeenLastCalledWith({ kind: "browser" }); + expect(current.client.sources).not.toHaveBeenCalled(); + + current.isClientActivated.mockReturnValue(true); + await current.openPicker(); + expect(current.setNativeSources).toHaveBeenLastCalledWith(expect.objectContaining({ kind: "ready" })); + expect(current.client.sources).toHaveBeenCalledOnce(); + + current.context.phase = "live"; + await current.openPicker(); + expect(current.setNativeSources).toHaveBeenLastCalledWith({ kind: "browser" }); + expect(current.client.sources).toHaveBeenCalledOnce(); + }); + it("shows App incompatibility in the picker and clears it on a successful refresh", async () => { const current = fixture(); current.context.phase = "idle"; From b5d710f3757674686b5c408953d26f048150c2e9 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:28:11 +0800 Subject: [PATCH 166/262] fix(ice): preserve STUN destination provenance for native prediction --- docs/research/nat-traversal.md | 22 ++++++- internal/app/mediaedge/candidate_gathering.go | 4 +- .../app/mediaedge/candidate_gathering_test.go | 62 +++++++++++++++++++ internal/app/mediaedge/ice_socket.go | 7 ++- src/client/native/native-sender-edge.ts | 2 +- src/client/native/native-viewer-peer.ts | 17 ++--- src/client/webrtc/nat-prediction.ts | 12 +++- tests/nat-prediction.test.ts | 14 ++++- tests/native-sender-edge.test.ts | 24 ++++++- tests/native-viewer-peer.test.ts | 46 ++++++++++++++ 10 files changed, 190 insertions(+), 20 deletions(-) diff --git a/docs/research/nat-traversal.md b/docs/research/nat-traversal.md index 23d2e6b4f..45f023607 100644 --- a/docs/research/nat-traversal.md +++ b/docs/research/nat-traversal.md @@ -43,10 +43,30 @@ discovery failure. The pinned Pion STUN client also received and decoded `XOR-MAPPED-ADDRESS` from each destination; the fallback is not Browser-only. Native regression checks confirm a healthy destination returns without waiting for silent UDP or unresolved DNS, and cancellation retires the -survey. These controls do not reproduce the reporter's network or establish its +survey. A Browser control with two failing DNS destinations still exposes the +healthy MiWiFi result after 55 ms. A bound, deliberately silent UDP fixture +beside a healthy fixture exposes the latter's synthetic srflx result after 4 ms, +while gathering remains incomplete at 15 seconds. Closed-port controls did not +produce the same result; a closed port is not a silent-drop fixture. The +[upstream candidate path](https://webrtc.googlesource.com/src/+/refs/heads/main/p2p/base/stun_port.cc) +emits successful mappings separately from gathering completion. These controls +do not reproduce the reporter's network or establish its final connection cause. The [WebRTC event specification](https://www.w3.org/TR/webrtc/#rtcicecandidateerrorevent) owns error semantics; compare the attempt's gathered and selected candidates too. +The follow-up also found a separate Native provenance gap on Sites with extra +ordinary STUN destinations. All `nsN` results were eligible for prediction, +while Browser observations were restricted to the configured three-URL survey. +Native candidate markers now identify their requested destination, independent +of response order. Both Native adapters filter prediction evidence against the +same survey set and still forward every ordinary candidate. The Viewer retains +the offer's actual destination list with its candidate batch, so a later config +update cannot reinterpret an in-flight response. Controlled tests cover +out-of-order results, an unrelated fourth destination, repeated mappings and +stalled DNS. +This does not identify the public-invitation report's cause, whose configured +set already contains exactly three destinations. + ### SFU Behind Server NAT An isolated Linux kernel-NAT control on 2026-10-09 places the current Server diff --git a/internal/app/mediaedge/candidate_gathering.go b/internal/app/mediaedge/candidate_gathering.go index ef94a6fc6..fbce646db 100644 --- a/internal/app/mediaedge/candidate_gathering.go +++ b/internal/app/mediaedge/candidate_gathering.go @@ -117,7 +117,6 @@ func (gathering *localCandidateGathering) start() { } }) }() - index := 0 var observed []mappedAddress for survey != nil || mapping != nil { select { @@ -135,11 +134,10 @@ func (gathering *localCandidateGathering) start() { survey = nil continue } - index++ mid := "0" line := uint16(0) gathering.emitCandidate(&webrtc.ICECandidateInit{ - Candidate: "candidate:ns" + strconv.Itoa(index) + + Candidate: "candidate:ns" + strconv.Itoa(mapped.serverIndex) + " 1 udp " + strconv.Itoa(surveyCandidatePriority) + " " + mapped.address + " " + strconv.Itoa(mapped.port) + " typ srflx raddr 0.0.0.0 rport " + diff --git a/internal/app/mediaedge/candidate_gathering_test.go b/internal/app/mediaedge/candidate_gathering_test.go index 2c8a132a8..cb484af6f 100644 --- a/internal/app/mediaedge/candidate_gathering_test.go +++ b/internal/app/mediaedge/candidate_gathering_test.go @@ -4,10 +4,12 @@ import ( "context" "net" "strings" + "sync/atomic" "testing" "time" "github.com/TNTcraftHIM/Piik/internal/diagnostics" + "github.com/pion/stun/v4" "github.com/pion/turn/v5" "github.com/pion/webrtc/v4" ) @@ -135,6 +137,9 @@ func TestSurveyHealthySTUNDoesNotWaitForSlowDNS(t *testing.T) { if mapped.port != socket.localPort { t.Fatal("STUN did not use the shared media socket") } + if mapped.serverIndex != 2 { + t.Fatal("failed DNS shifted the responding destination's identity") + } case <-time.After(time.Second): t.Fatal("healthy STUN waited for the broken DNS target") } @@ -148,6 +153,63 @@ func TestSurveyHealthySTUNDoesNotWaitForSlowDNS(t *testing.T) { } } +func TestSurveyCandidateIdentitySurvivesResponseOrder(t *testing.T) { + engine, err := NewEngine(EngineOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = engine.Close() }) + socket := testICESocket(t, engine) + var releaseSlow atomic.Bool + slow := bindingServer(t, func(request *stun.Message, sender *net.UDPAddr, listener *net.UDPConn) { + if releaseSlow.Load() { + answerBinding(request, sender, listener, 40000) + } + }) + fast := bindingServer(t, func(request *stun.Message, sender *net.UDPAddr, listener *net.UDPConn) { + answerBinding(request, sender, listener, 40003) + }) + candidates := make(chan *webrtc.ICECandidateInit, 4) + gathering := newLocalCandidateGathering(socket, []webrtc.ICEServer{{URLs: []string{ + "turn:unused.invalid:3478", "stun:" + slow.String(), "stun:" + fast.String(), + }}}, nil, func(candidate *webrtc.ICECandidateInit) { candidates <- candidate }) + t.Cleanup(gathering.close) + gathering.start() + for _, want := range []struct{ foundation, port string }{{"candidate:ns2", "40003"}, {"candidate:ns1", "40000"}} { + select { + case candidate := <-candidates: + if candidate == nil { + t.Fatal("survey ended before both destinations responded") + } + fields := strings.Fields(candidate.Candidate) + if fields[0] != want.foundation || fields[5] != want.port { + t.Fatalf("candidate lost its destination: %s; want %+v", candidate.Candidate, want) + } + releaseSlow.Store(true) + case <-time.After(2 * time.Second): + t.Fatal("STUN response did not arrive") + } + } +} + +func TestSurveyDuplicateMappingRemainsOneOrdinaryCandidate(t *testing.T) { + engine, err := NewEngine(EngineOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = engine.Close() }) + socket := testICESocket(t, engine) + servers := []webrtc.ICEServer{{URLs: []string{localSurveyServer(t), localSurveyServer(t)}}} + var observations []mappedAddress + socket.surveySTUN(t.Context(), servers, func(mapped mappedAddress) { + observations = append(observations, mapped) + }) + if len(observations) != 1 || observations[0].port != socket.localPort || + (observations[0].serverIndex != 1 && observations[0].serverIndex != 2) { + t.Fatalf("duplicate destination mappings became independent evidence: %+v", observations) + } +} + func TestSurveyReturnsWhenEveryDestinationAlreadyFailed(t *testing.T) { probe, err := net.ListenUDP("udp6", &net.UDPAddr{IP: net.IPv6loopback}) if err != nil { diff --git a/internal/app/mediaedge/ice_socket.go b/internal/app/mediaedge/ice_socket.go index 5c5ff9798..3730dbe98 100644 --- a/internal/app/mediaedge/ice_socket.go +++ b/internal/app/mediaedge/ice_socket.go @@ -101,6 +101,8 @@ func (socket *iceSocket) close() { type mappedAddress struct { address string port int + // One-based position in the requested STUN list, independent of response order. + serverIndex int } func (socket *iceSocket) surveySTUN( @@ -114,8 +116,11 @@ func (socket *iceSocket) surveySTUN( resolver := net.DefaultResolver results := make(chan mappedAddress) var pending sync.WaitGroup + index := 0 for _, server := range servers { for _, rawURL := range server.URLs { + index++ + serverIndex := index uri, err := stun.ParseURI(rawURL) if err != nil || uri.Scheme != stun.SchemeTypeSTUN || uri.Proto != stun.ProtoTypeUDP || uri.Port < 1 || uri.Port > 65_535 { @@ -144,7 +149,7 @@ func (socket *iceSocket) surveySTUN( "localPort", socket.localPort, "mappedAddress", diagnostics.ID(mapped.IP.String()), "mappedPort", mapped.Port, "durationMs", time.Since(started).Milliseconds()) select { - case results <- mappedAddress{address: mapped.IP.String(), port: mapped.Port}: + case results <- mappedAddress{address: mapped.IP.String(), port: mapped.Port, serverIndex: serverIndex}: case <-surveyContext.Done(): } }() diff --git a/src/client/native/native-sender-edge.ts b/src/client/native/native-sender-edge.ts index fccc72ebe..580527911 100644 --- a/src/client/native/native-sender-edge.ts +++ b/src/client/native/native-sender-edge.ts @@ -158,7 +158,7 @@ export class NativeSenderEdge { if (event.candidate) { this.localIceCandidates.add( event.candidate, - isNativeNatSurveyCandidate(event.candidate), + isNativeNatSurveyCandidate(event.candidate, this.iceConfig.iceServers, this.iceConfig.natPredictionStunUrls), ); } else { this.localIceCandidates.complete(); diff --git a/src/client/native/native-viewer-peer.ts b/src/client/native/native-viewer-peer.ts index 2827efc79..7ef98690d 100644 --- a/src/client/native/native-viewer-peer.ts +++ b/src/client/native/native-viewer-peer.ts @@ -196,7 +196,7 @@ class NativeViewerPeer implements ViewerMediaPeer { private parentPeerId: string | null = null; private connectionId: string | null = null; private bridge: NativeMediaBridge | null = null; - private localCandidates: NatPredictionCandidateBatch | null = null; + private localCandidates: { batch: NatPredictionCandidateBatch; iceConfig: IceConfig } | null = null; private unsubscribe: (() => void) | null = null; private unsubscribeClose: (() => void) | null = null; private snapshot: PeerSnapshot | null = null; @@ -336,7 +336,7 @@ class NativeViewerPeer implements ViewerMediaPeer { this.disposed = true; this.clearTimers(); this.disposeBridge(); - this.localCandidates?.discard(); + this.localCandidates?.batch.discard(); this.localCandidates = null; this.unsubscribe?.(); this.unsubscribe = null; @@ -366,7 +366,7 @@ class NativeViewerPeer implements ViewerMediaPeer { if (identityChanged) { const previous = this.connectionId; this.disposeBridge(); - this.localCandidates?.discard(); + this.localCandidates?.batch.discard(); if (previous) { await this.client.closeReceiver(this.sessionId, previous).catch( () => undefined, @@ -377,7 +377,7 @@ class NativeViewerPeer implements ViewerMediaPeer { } else if (!retainSource) { this.disposeBridge(); } - this.localCandidates?.discard(); + this.localCandidates?.batch.discard(); if (!retainSource) { this.state = "connecting"; this.failureNotified = false; @@ -427,7 +427,7 @@ class NativeViewerPeer implements ViewerMediaPeer { const candidates = predictionEnabled ? new NatPredictionCandidateBatch(sendCandidate) : null; - this.localCandidates = candidates; + this.localCandidates = candidates ? { batch: candidates, iceConfig: nativeIceConfig } : null; this.emit(); const pending = { bridge: retainSource ? this.bridge : null, bridgeFailed: false }; this.pendingOffer = pending; @@ -531,12 +531,13 @@ class NativeViewerPeer implements ViewerMediaPeer { } if (event.type === "edge-candidate") { if (this.localCandidates) { + const { batch, iceConfig } = this.localCandidates; if (event.candidate) { - this.localCandidates.add( + batch.add( event.candidate, - isNativeNatSurveyCandidate(event.candidate), + isNativeNatSurveyCandidate(event.candidate, iceConfig.iceServers, iceConfig.natPredictionStunUrls), ); - } else this.localCandidates.complete(); + } else batch.complete(); } else if (this.parentPeerId && this.connectionId) { this.events.sendSignal(this.parentPeerId, { kind: "candidate", diff --git a/src/client/webrtc/nat-prediction.ts b/src/client/webrtc/nat-prediction.ts index 9cfc83ea6..4df872efd 100644 --- a/src/client/webrtc/nat-prediction.ts +++ b/src/client/webrtc/nat-prediction.ts @@ -10,8 +10,18 @@ type ConcreteSignalCandidate = NonNullable; export function isNativeNatSurveyCandidate( candidate: SignalCandidate, + iceServers: readonly RTCIceServer[] | undefined, + auxiliaryStunUrls: readonly string[] = [], ): boolean { - return candidate !== null && /^candidate:ns\d+\s/i.test(candidate.candidate); + const index = candidate && /^candidate:ns([1-9]\d*)\s/i.exec(candidate.candidate)?.[1]; + if (!index) return false; + // App identifies the requested destination, never its response order. Extra + // ordinary STUN servers remain useful candidates but are not survey evidence. + const urls = iceServersWithNatPrediction(iceServers, true, auxiliaryStunUrls) + .flatMap(urlsOf).filter(url => /^stun:/i.test(url)); + const url = urls[Number(index) - 1]; + const normalized = url ? normalizedStunUrl(url) : null; + return normalized !== null && natPredictionSurveyUrls(iceServers, auxiliaryStunUrls).has(normalized); } const BASE_STUN_PORT = 3478; diff --git a/tests/nat-prediction.test.ts b/tests/nat-prediction.test.ts index bd9587c41..bc27bc27c 100644 --- a/tests/nat-prediction.test.ts +++ b/tests/nat-prediction.test.ts @@ -45,6 +45,10 @@ function rtcCandidate(port: number): RTCIceCandidate { describe("NAT prediction ICE adapter", () => { it("uses only explicit Native survey candidates as prediction evidence", () => { + const iceConfig = { + iceServers: [{ urls: ["stun:share.example.test:3478", "stun:other.example.test:3478"] }], + natPredictionStunUrls: ["stun:share.example.test:3479", "stun:share.example.test:3480"], + }; const survey = candidate(40_000); if (survey) { survey.candidate = survey.candidate.replace( @@ -53,9 +57,13 @@ describe("NAT prediction ICE adapter", () => { ); } - expect(isNativeNatSurveyCandidate(survey)).toBe(true); - expect(isNativeNatSurveyCandidate(candidate(40_000))).toBe(false); - expect(isNativeNatSurveyCandidate(hostCandidate())).toBe(false); + const eligible = (value: SignalCandidate) => isNativeNatSurveyCandidate(value, iceConfig.iceServers, iceConfig.natPredictionStunUrls); + expect(eligible(survey)).toBe(true); + for (const [index, expected] of [[2, false], [3, true], [4, true], [5, false], [0, false]] as const) { + expect(eligible({ candidate: `candidate:ns${index} 1 udp 1 203.0.113.7 40000 typ srflx` })).toBe(expected); + } + expect(eligible(candidate(40_000))).toBe(false); + expect(eligible(hostCandidate())).toBe(false); }); it("keeps unexpected ordinary-candidate failures visible", async () => { diff --git a/tests/native-sender-edge.test.ts b/tests/native-sender-edge.test.ts index 4abe7b015..218844124 100644 --- a/tests/native-sender-edge.test.ts +++ b/tests/native-sender-edge.test.ts @@ -7,7 +7,7 @@ import { } from "../src/client/native/native-sender-edge"; import type { NativeClientEvent } from "../src/client/native/wire"; -function fixture(natPrediction = false) { +function fixture(natPrediction = false, extraStun = false) { let listener: ((event: NativeClientEvent) => void) | null = null; const prepareEdge = vi.fn(async () => ({ type: "offer", @@ -37,7 +37,7 @@ function fixture(natPrediction = false) { "share_1234567", natPrediction ? { - iceServers: [{ urls: "stun:share.example.test:3478" }], + iceServers: [{ urls: ["stun:share.example.test:3478", ...(extraStun ? ["stun:other.example.test:3478"] : [])] }], natPredictionStunUrls: [ "stun:share.example.test:3479", "stun:share.example.test:3480", @@ -170,4 +170,24 @@ describe("native sender edge adapter", () => { expect(candidates.filter((candidate) => /^candidate:s[pm]\d+ /.test(candidate))) .toHaveLength(8); }); + + it("forwards extra STUN candidates without mistaking them for prediction evidence", async () => { + const current = fixture(true, true); + expect(await current.edge.start()).toBe(true); + const emit = (index: number, port: number) => current.emit({ + version: 10, type: "edge-candidate", shareId: "share_1234567", connectionId: "edge_12345678", + candidate: { candidate: `candidate:ns${index} 1 udp 1 203.0.113.7 ${port} typ srflx` }, + }); + const candidates = () => current.sent.flatMap(payload => payload.kind === "candidate" && payload.candidate + ? [payload.candidate.candidate] : []); + // Deliberately receive out of list order, with three seemingly sequential ports. + emit(3, 40_006); + emit(2, 40_003); + emit(1, 40_000); + expect(candidates()).toHaveLength(3); + emit(4, 40_012); + expect(candidates().filter(value => /^candidate:ns/.test(value))).toHaveLength(4); + expect(candidates().filter(value => /^candidate:s[pm]/.test(value))).toHaveLength(8); + current.edge.dispose(); + }); }); diff --git a/tests/native-viewer-peer.test.ts b/tests/native-viewer-peer.test.ts index 62fd3b554..4643467f6 100644 --- a/tests/native-viewer-peer.test.ts +++ b/tests/native-viewer-peer.test.ts @@ -16,6 +16,52 @@ afterEach(() => { vi.unstubAllGlobals(); }); +it("uses the offer's STUN destinations for prediction while the next ICE config changes", async () => { + vi.useFakeTimers(); + vi.stubGlobal("window", globalThis); + vi.stubGlobal("MediaStream", class { getTracks() { return []; } }); + vi.stubGlobal("RTCPeerConnection", class { close() {} }); + vi.spyOn(NativeMediaBridge.prototype, "start").mockResolvedValue({} as MediaStream); + let onEvent!: (event: NativeClientEvent) => void; + const client = { + receiveOffer: vi.fn(async () => ({ answer: { type: "answer", sdp: "v=0\r\n" }, audio: false, codec: "vp8" })), + onEvent: (handler: typeof onEvent) => { onEvent = handler; return () => {}; }, + onClose: () => () => {}, + closeReceiver: vi.fn(async () => {}), closeEdge: vi.fn(async () => {}), + } as unknown as NativeClient; + const original = { + iceServers: [{ urls: ["stun:site.example:3478", "stun:other.example:3478"] }], + natPredictionStunUrls: ["stun:site.example:3479", "stun:site.example:3480"], + }; + const sent: SignalPayload[] = []; + const peer = new NativeCapableViewerPeer(original, { + sendSignal: (_parent, payload) => { sent.push(payload); return true; }, + sendRestartRequest: () => true, onStream: () => {}, onUpdate: () => {}, + }, { natPredictionEnabled: true }, async () => client, () => {}, "session", 2); + const emit = (index: number, port: number) => onEvent({ + version: 10, type: "edge-candidate", shareId: "session", connectionId: "current", + candidate: { candidate: `candidate:ns${index} 1 udp 1 203.0.113.7 ${port} typ srflx` }, + }); + const candidates = () => sent.flatMap(payload => payload.kind === "candidate" ? [payload.candidate] : []); + try { + await peer.acceptSignal("parent", { + kind: "description", connectionId: "current", + description: { type: "offer", sdp: "v=0\r\nm=video 9 UDP/TLS/RTP/SAVPF 96\r\na=sendonly\r\na=rtpmap:96 VP8/90000\r\n" }, + }); + peer.updateIceConfig({ + ...original, iceServers: [{ urls: ["stun:other.example:3478", "stun:site.example:3478"] }], + }); + emit(3, 40006); + emit(2, 40003); // Ordinary extra destination must not complete a predicted sequence. + emit(1, 40000); + expect(candidates()).toHaveLength(3); + emit(4, 40012); + expect(candidates()).toHaveLength(12); // Four ordinary candidates plus the bounded eight predictions. + } finally { + peer.dispose(); + } +}); + it("retires queued candidates when disposed during their delivery", async () => { vi.stubGlobal("window", globalThis); vi.stubGlobal("MediaStream", class { getTracks() { return []; } }); From a38679c3d05a7b07687d80ec7d5b79e7443c7f2b Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:28:29 +0800 Subject: [PATCH 167/262] test(media): isolate recovery capacity from host UDP buffering --- internal/app/mediaedge/relay_input_test.go | 34 +++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/internal/app/mediaedge/relay_input_test.go b/internal/app/mediaedge/relay_input_test.go index b77c65a2c..f7a792fdf 100644 --- a/internal/app/mediaedge/relay_input_test.go +++ b/internal/app/mediaedge/relay_input_test.go @@ -10,8 +10,12 @@ import ( "github.com/TNTcraftHIM/Piik/internal/app/nativecapture" "github.com/TNTcraftHIM/Piik/internal/media/encoded" "github.com/livekit/livekit-server/pkg/sfu" + "github.com/pion/ice/v4" + "github.com/pion/interceptor" + "github.com/pion/logging" "github.com/pion/rtp" "github.com/pion/rtp/codecs" + "github.com/pion/transport/v5/vnet" "github.com/pion/webrtc/v4" ) @@ -178,9 +182,27 @@ func TestLargeRecoveryUsesTheDefaultPacketWindow(t *testing.T) { if len(packets) <= 500 || len(packets) > encoded.MaxPacketWindow { t.Fatal("fixture does not exercise the packet window boundary") } + // Isolate packet-window capacity from host UDP receive-buffer overflow. + // The production sender, SRTP transport and frame assembler still run here. + router, err := vnet.NewRouter(&vnet.RouterConfig{CIDR: "10.0.0.0/24", + QueueSize: encoded.MaxPacketWindow, LoggerFactory: logging.NewDefaultLoggerFactory()}) + check(err) + sendNet, err := vnet.NewNet(&vnet.NetConfig{StaticIPs: []string{"10.0.0.1"}}) + check(err) + receiveNet, err := vnet.NewNet(&vnet.NetConfig{StaticIPs: []string{"10.0.0.2"}}) + check(err) + check(router.AddNet(sendNet)) + check(router.AddNet(receiveNet)) + check(router.Start()) + defer router.Stop() engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true, InitialBitrate: 12_000_000}) check(err) defer engine.Close() + engine.listenPacket = sendNet.ListenPacket + engine.bindAddress = "10.0.0.1:0" + engine.settings.SetNet(sendNet) + engine.settings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + engine.settings.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled) source, err := engine.NewSource("h264", 1, 1, nil) check(err) defer source.Close() @@ -189,7 +211,17 @@ func TestLargeRecoveryUsesTheDefaultPacketWindow(t *testing.T) { edge, err := engine.NewEdge(source, EdgeOptions{ConnectionID: "large-recovery"}) check(err) defer edge.Close() - receiver := newReceiver(t) + media := &webrtc.MediaEngine{} + check(media.RegisterDefaultCodecs()) + registry := &interceptor.Registry{} + check(webrtc.RegisterDefaultInterceptors(media, registry)) + settings := webrtc.SettingEngine{} + settings.SetNet(receiveNet) + settings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + settings.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled) + receiver, err := webrtc.NewAPI(webrtc.WithMediaEngine(media), webrtc.WithInterceptorRegistry(registry), + webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) + check(err) defer receiver.Close() var complete atomic.Bool receiver.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { From 26827dbd755f7345a6c1456ed9b23a063f211292 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:42:42 +0800 Subject: [PATCH 168/262] fix(app): restore launcher settings after a failed initial read --- src/client/pages/AppLauncherPage.tsx | 11 +-- tests/app-launcher-recovery.test.ts | 101 +++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 tests/app-launcher-recovery.test.ts diff --git a/src/client/pages/AppLauncherPage.tsx b/src/client/pages/AppLauncherPage.tsx index a3cefeb35..2e757fdde 100644 --- a/src/client/pages/AppLauncherPage.tsx +++ b/src/client/pages/AppLauncherPage.tsx @@ -74,8 +74,9 @@ export function AppLauncherPage() { let current = true; let timer: ReturnType | undefined; let checkedRelease = false; + let restoredSettings = false; let reading = false; - const readState = (restoreSettings = false) => { + const readState = () => { if (reading || !current) return; reading = true; clearTimeout(timer); @@ -86,10 +87,11 @@ export function AppLauncherPage() { }) .then((state) => { if (!current) return; - if (restoreSettings) { + if (!restoredSettings) { setMode(state.defaultMode); setLocalAccessPassword(state.localAccessPassword); setLan(state.lan); + restoredSettings = true; } setVersion(state.version); setTarget(state.target); @@ -97,7 +99,8 @@ export function AppLauncherPage() { setSite(state.site); setAppDebug(state.debug); setDebug(state.debug === true); - if (state.detail) setError({ kind: "launch", detail: state.detail }); + setError(previous => state.detail ? { kind: "launch", detail: state.detail } + : previous?.kind === "load" ? null : previous); setLoading(false); if (state.starting) timer = setTimeout(() => void readState(), 750); if (checkedRelease) return; @@ -118,7 +121,7 @@ export function AppLauncherPage() { setLoading(false); }).finally(() => { reading = false; }); }; - void readState(true); + void readState(); const refresh = () => { void readState(); }; window.addEventListener("focus", refresh); return () => { diff --git a/tests/app-launcher-recovery.test.ts b/tests/app-launcher-recovery.test.ts new file mode 100644 index 000000000..10cbd855c --- /dev/null +++ b/tests/app-launcher-recovery.test.ts @@ -0,0 +1,101 @@ +import { readFileSync } from "node:fs"; +import { runInNewContext } from "node:vm"; +import ts from "typescript"; +import { describe, expect, it, vi } from "vitest"; +import { z } from "zod"; + +// Run the page's actual state-read effect, including cleanup, without a DOM. +const source = ts.createSourceFile("AppLauncherPage.tsx", readFileSync( + new URL("../src/client/pages/AppLauncherPage.tsx", import.meta.url), "utf8", +), ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX); +let schema: string | undefined; +let effect: string | undefined; +function collect(node: ts.Node): void { + if (ts.isVariableDeclaration(node) && node.name.getText(source) === "launcherStateSchema") { + schema = `const ${node.getText(source)};`; + } + if (ts.isCallExpression(node) && node.expression.getText(source) === "useEffect" + && node.arguments[1]?.getText(source) === "[reload]") { + effect = node.arguments[0]!.getText(source); + } + ts.forEachChild(node, collect); +} +collect(source); +if (!schema || !effect) throw new Error("Missing App launcher state-read owner"); +const executable = ts.transpileModule(`${schema}\n(${effect})();`, { + compilerOptions: { target: ts.ScriptTarget.ES2022 }, +}).outputText; +const settle = () => new Promise(resolve => setImmediate(resolve)); +const saved = { + site: "https://site.example", localAccessPassword: "saved password", defaultMode: "local", + revision: "test", version: "development", debug: false, + lan: { addresses: [{ address: "192.0.2.1", name: "LAN" }], selected: "192.0.2.1" }, +}; +const response = (state: typeof saved & { detail?: string } = saved) => ({ ok: true, json: async () => state }); + +function mount(fetch: ReturnType) { + const state: Record = {}; + const setters = Object.fromEntries([ + "Mode", "LocalAccessPassword", "Lan", "Version", "Target", "Starting", "Site", + "AppDebug", "Debug", "Error", "Loading", "Update", + ].map(name => [`set${name}`, vi.fn((value: unknown) => { + state[name] = typeof value === "function" ? value(state[name]) : value; + })])); + const window = new EventTarget(); + const checkReleaseUpdate = vi.fn(async () => null); + const cleanup = runInNewContext(executable, { + ...setters, z, fetch, window, checkReleaseUpdate, setTimeout, clearTimeout, + }) as () => void; + return { state, setters, cleanup, checkReleaseUpdate, focus: () => window.dispatchEvent(new Event("focus")) }; +} + +describe("App launcher state recovery", () => { + it("restores saved choices after a failed first read, then preserves unsaved edits on focus", async () => { + const fetch = vi.fn(async () => response()).mockRejectedValueOnce(new Error("unavailable")); + const page = mount(fetch); + await settle(); + expect(page.state.Error).toEqual({ kind: "load" }); + page.focus(); + await settle(); + expect(page.state).toMatchObject({ + Error: null, Loading: false, Site: saved.site, Mode: "local", + LocalAccessPassword: saved.localAccessPassword, Lan: saved.lan, + }); + Object.assign(page.state, { Mode: "link", LocalAccessPassword: "edited", Lan: { selected: "192.0.2.2" } }); + page.focus(); + await settle(); + expect(page.state).toMatchObject({ Mode: "link", LocalAccessPassword: "edited", Lan: { selected: "192.0.2.2" } }); + expect(fetch).toHaveBeenCalledTimes(3); + expect(page.checkReleaseUpdate).toHaveBeenCalledOnce(); + page.cleanup(); + }); + + it("preserves launch errors on successful reads and accepts the App's current failure detail", async () => { + const fetch = vi.fn(async () => response()); + const page = mount(fetch); + await settle(); + page.state.Error = { kind: "launch", detail: "launch request failed" }; + page.focus(); + await settle(); + expect(page.state.Error).toEqual({ kind: "launch", detail: "launch request failed" }); + fetch.mockResolvedValueOnce(response({ ...saved, detail: "App runtime failed" })); + page.focus(); + await settle(); + expect(page.state.Error).toEqual({ kind: "launch", detail: "App runtime failed" }); + page.cleanup(); + }); + + it("ignores an in-flight response and future focus after retirement", async () => { + let complete!: (value: ReturnType) => void; + const fetch = vi.fn(() => new Promise>(resolve => { complete = resolve; })); + const page = mount(fetch); + page.cleanup(); + complete(response()); + await settle(); + page.focus(); + expect(fetch).toHaveBeenCalledOnce(); + expect(page.state).toEqual({}); + expect(page.checkReleaseUpdate).not.toHaveBeenCalled(); + for (const setter of Object.values(page.setters)) expect(setter).not.toHaveBeenCalled(); + }); +}); From 9dc8e9aea66146f54c7a9c7e9e714df73c6c0e60 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:54:09 +0800 Subject: [PATCH 169/262] fix(site): align App guidance and film with passive connection --- cmd/piik-app/README.md | 2 +- cmd/piik-app/README.zh-CN.md | 2 +- docs/guide/troubleshooting.md | 5 +++-- docs/guide/troubleshooting.zh-CN.md | 3 ++- scripts/check-website-film.js | 13 ++++++++----- site/film/README.md | 8 +++++++- site/film/ui/main.tsx | 8 +++----- 7 files changed, 25 insertions(+), 16 deletions(-) diff --git a/cmd/piik-app/README.md b/cmd/piik-app/README.md index 72a4bec6c..2e4e8f874 100644 --- a/cmd/piik-app/README.md +++ b/cmd/piik-app/README.md @@ -146,7 +146,7 @@ go build -o build/dev/piik-app.exe ./cmd/piik-app ./build/dev/piik-app.exe ``` -These commands build the App only. Choose **Local room** or a configured Site +These commands build the App only. Choose **Local room** or open a Piik Site for Browser capture. Native capture and **Public invite** need their helpers; use `--capture-process` / `--tunnel-process` to select built helpers, or follow [packaging](#packaging) for a complete bundle. diff --git a/cmd/piik-app/README.zh-CN.md b/cmd/piik-app/README.zh-CN.md index f92de74b9..894159263 100644 --- a/cmd/piik-app/README.zh-CN.md +++ b/cmd/piik-app/README.zh-CN.md @@ -118,7 +118,7 @@ go build -o build/dev/piik-app.exe ./cmd/piik-app ./build/dev/piik-app.exe ``` -这些命令只构建 App。体验浏览器采集时,选择 **本地房间** 或配置好的站点。 +这些命令只构建 App。体验浏览器采集时,选择 **本地房间**,或直接打开 Piik 站点。 原生采集和 **公网邀请** 需要各自的辅助程序,可用 `--capture-process` / `--tunnel-process` 指定已构建的程序,或按[打包](#打包)步骤生成完整程序包。 diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 958201ccd..ce5234029 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -341,8 +341,9 @@ same invitation works there, focus on the original network's restrictions. For regular use, consider **self-hosting a Piik site with SFU media forwarding**. This gives viewers a server path when direct connections fail. Follow the [deployment guide](../operations/self-hosting.md#optional-media-fallback), including -its public-address and UDP-port requirements. The Host can then open that site -through the App's **Site** mode and send invitations from the new site. +its public-address and UDP-port requirements. Keep Piik App running, open the new +site in your browser and select **App** at the top to approve its local access. +Send invitations generated by that site. The public demo and App Public invite mode remain P2P-only. SFU also requires usable UDP: when the network blocks UDP entirely, use another network or ask the diff --git a/docs/guide/troubleshooting.zh-CN.md b/docs/guide/troubleshooting.zh-CN.md index cef7570bf..7b5464511 100644 --- a/docs/guide/troubleshooting.zh-CN.md +++ b/docs/guide/troubleshooting.zh-CN.md @@ -279,7 +279,8 @@ Piik 通过 WebRTC 传输画面和声音。禁用 WebRTC 或禁止未经代理 经常需要使用的话,建议考虑 **自建 Piik 站点并启用 SFU 媒体转发**,让直连失败时有服务器路线。 按[部署指南](../operations/self-hosting.zh-CN.md#可选启用媒体兜底)配置公网地址并放行所需 UDP 端口。 -房主随后可通过 App 的 **站点** 模式打开新站点,并发送该站点生成的邀请链接。 +保持 Piik App 运行,用浏览器打开新站点,点击顶部的 **App** 完成本机授权, +再发送该站点生成的邀请链接。 公开 Demo 和 App 的公网邀请模式只有 P2P。SFU 同样需要 UDP 可用;如果网络完全禁止 UDP, 仍需换网络或联系管理员放行,单独搭建服务器不能解除这类限制。 diff --git a/scripts/check-website-film.js b/scripts/check-website-film.js index 91fdc1af8..a083f8fc3 100644 --- a/scripts/check-website-film.js +++ b/scripts/check-website-film.js @@ -1,4 +1,4 @@ -// Run in a freshly opened local /film/ page using agent-browser eval --stdin. +// Bundle this check, then run it in a fresh local /film/ page with agent-browser. // This exercises the real document/module; it does not mock its scene renderer. // First click #language with agent-browser to grant native audio user activation. (async () => { @@ -63,7 +63,7 @@ seek(time); assert(scene().includes(`scene-${name}`), `The product sequence must show ${name}`); } - const {BEAT,BAR,DOWNLOAD_AT,DURATION} = await import('./score.js'); + const {BEAT,BAR,DOWNLOAD_AT,DURATION} = await import('../site/film/score.js'); for (const [bar, feature] of [[19.5, 'free'], [20.5, 'p2p'], [21.5, 'encode'], [23, 'devices']]) { seek(bar * BAR); const visible = [...el('scene-features').children].filter(node => node.style.display !== 'none'); @@ -315,6 +315,9 @@ await at('website',0,lang); const downloadedAgain = await at('website',4.5*BEAT,lang); assert(Math.hypot(downloadedAgain.x-downloaded.x, downloadedAgain.y-downloaded.y)<.01, 'The website scroll and click must survive reverse seeking'); + await at('launch',0,lang); + assert(doc.querySelector('[role="radio"][aria-checked="true"]').getAttribute('aria-label') === + (lang === 'en' ? 'Public invite' : '公网邀请'), 'The launcher must start with Public invite selected'); await at('launch',5*BEAT,lang); assert(doc.querySelector('.lr-room') && !doc.querySelector('.lr-room-chip'), 'Opening Piik must show the ready page before a room has been created'); await at('share',3*BEAT,lang); @@ -338,12 +341,12 @@ 'The viewer wide shot must include current playback, room interactions and the room code'); for (const [scene,beat,selector] of [ ['desktop',2.5,'#desktop-app'], - ['launch',1.5,'[role="radio"]'], ['launch',4,'button[type="submit"]'], + ['launch',1.5,'[role="radio"][aria-checked="true"]'], ['launch',4,'button[type="submit"]'], ['share',1.5,'.lr-entry-action button'], ['share',4,'.lr-source-option'], ['invite',1.5,'.lr-invite-url'], ['invite',4,'#chat-draft'], ['invite',6,'#chat-send'], ]) { const p = await at(scene,beat*BEAT,lang); - const element = doc.querySelectorAll(selector)[selector==='[role="radio"]'?1:0]; + const element = doc.querySelector(selector); const target = selector==='.lr-invite-url' ? element.closest('.lr-row').querySelector('button') : element; const b = target.getBoundingClientRect(); assert(p.visible && p.x>=b.left && p.x<=b.right && p.y>=b.top && p.y<=b.bottom, 'Clicks must land on visible controls'); @@ -385,7 +388,7 @@ document.body.append(hero); try { await loaded; - const {HERO_KINDS,HERO_CUT,heroFrame} = await import('../assets/activities.js'); + const {HERO_KINDS,HERO_CUT,heroFrame} = await import('../site/assets/activities.js'); const doc = hero.contentDocument, win = hero.contentWindow; assert(!doc.getElementById('montage-cheer'), 'The homepage and README hero must omit the film reaction caption'); const animations = doc.getAnimations(); diff --git a/site/film/README.md b/site/film/README.md index e8c78440e..73606d9cb 100644 --- a/site/film/README.md +++ b/site/film/README.md @@ -116,10 +116,16 @@ ffmpeg -i Funkorama.mp3 -vn -t 76.039604 \ ## Focused check +Bundle the check's source helpers locally; production serves hashed assets: + +```sh +npx esbuild scripts/check-website-film.js --bundle --format=esm --minify --outfile=build/check-website-film.js +``` + Open a fresh local film page in a named `agent-browser` session and run `agent-browser --session click '#language'` to grant the user gesture required by native audio. Then evaluate -[`scripts/check-website-film.js`](../../scripts/check-website-film.js) using +the resulting `build/check-website-film.js` using `agent-browser --session eval --stdin`. The browser check covers lazy audio, playback/seek/replay, language links, obsolete play promises, native media pause, audio failure/ending, bilingual headline bounds, unobscured scenario diff --git a/site/film/ui/main.tsx b/site/film/ui/main.tsx index a4108cea1..52abd1713 100644 --- a/site/film/ui/main.tsx +++ b/site/film/ui/main.tsx @@ -47,7 +47,7 @@ import { point } from "./cursor"; import "../../../src/client/styles.css"; import "./ui.css"; -type Shot = "desktop" | "local" | "link" | "idle" | "picker" | "host" | "copied" | "chat" | "draft" | "sent" | "viewer"; +type Shot = "desktop" | "link" | "idle" | "picker" | "host" | "copied" | "chat" | "draft" | "sent" | "viewer"; const noop = () => {}; const roomId = "9527"; const inviteUrl = `https://invite.piik.example/r/${roomId}`; @@ -179,7 +179,7 @@ function Screen({ shot }: { shot: Shot }) {
    ); - const launcher = shot === "local" || shot === "link"; + const launcher = shot === "link"; const chat = shot === "chat" || shot === "draft" || shot === "sent"; const live = shot === "host" || shot === "copied" || shot === "viewer" || chat; const hostStatus = deriveHostStatus({ @@ -436,8 +436,6 @@ window.addEventListener("message", (event) => { : data.scene === "launch" ? t >= cues.launch + BEAT / 2 ? "idle" - : t < cues.mode + BEAT / 4 - ? "local" : "link" : data.scene === "share" ? t < cues.share + BEAT / 4 @@ -513,7 +511,7 @@ function paint(data: Frame) { document.getElementById("desktop-app")!.style.transform = `scale(${1 - .04 * press})`; } if (data.scene === "launch") { - const mode = center(document.querySelectorAll('[role="radio"]')[1]); + const mode = center(document.querySelector('[role="radio"][aria-checked="true"]')); target = mode; clickAt = cues.mode; hideAt = cues.launch + BEAT / 2; From a6a5f15d19db10b6921f21866fc91acb5ba00009 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:42:20 +0800 Subject: [PATCH 170/262] fix(ui): align App entry controls with shared interaction feedback --- .../components/living/AppConnectionButton.tsx | 4 +- src/client/components/living/LauncherForm.tsx | 8 +- src/client/styles.css | 15 +- src/client/ui/icons.tsx | 2 +- tests/app-connection-button.test.ts | 148 ++++++++++++++++++ 5 files changed, 162 insertions(+), 15 deletions(-) create mode 100644 tests/app-connection-button.test.ts diff --git a/src/client/components/living/AppConnectionButton.tsx b/src/client/components/living/AppConnectionButton.tsx index c46705d39..7ab0d3aa5 100644 --- a/src/client/components/living/AppConnectionButton.tsx +++ b/src/client/components/living/AppConnectionButton.tsx @@ -68,9 +68,9 @@ export function AppConnectionButton({ sourcePicker = false, onConnected }: { const label = t(state === "incompatible" ? "native.incompatible" : state === "unavailable" ? "client.connect.unavailable" : sourcePicker ? "client.connect.captureHint" : "client.connect.action"); - return -

    - {target ? - {vis ? null : t("client.launch.go")} + {target ? + {vis ? null : {t("client.launch.go")}} : <>

    {t("client.launch.room")}

    - + {t("client.launch.accessSettings")} {localAccessPassword && @@ -181,7 +181,7 @@ export function LauncherForm({
    {mode === "local" && lan && (lan.addresses.length !== 1 || !lan.selected) ? (
    - + {t("client.launch.lanAddress")} diff --git a/src/client/styles.css b/src/client/styles.css index 5600dd34a..30c391d2c 100644 --- a/src/client/styles.css +++ b/src/client/styles.css @@ -546,7 +546,7 @@ body.lr-theater-open .lr-top { visibility: hidden; } .lr-source-picker-body > [role="status"] { text-align: center; } .lr-source-app-connection { display: grid; justify-items: center; gap: 6px; margin-top: 12px; } .lr-source-app-connection small { color: #9aa8bb; font-size: 12px; text-align: center; } -.lr-source-app-connect { display: flex; gap: 8px; width: auto; padding: 0 12px; margin: 0; } +.lr-source-app-connect { display: flex; gap: 8px; width: auto; padding: 0 12px; margin: 0; border-radius: 999px; } .lr-source-picker-list { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 240px), 1fr)); @@ -996,6 +996,7 @@ html[data-theme="dark"] .lr-invite-url { background: #1a2634; border-color: #334 background: var(--paper); color: var(--ink); white-space: nowrap; + text-decoration: none; box-shadow: var(--control-shadow); transition: transform 260ms var(--spring), background-color 140ms ease, border-color 140ms ease; } @@ -1008,6 +1009,7 @@ html[data-theme="dark"] .lr-invite-url { background: #1a2634; border-color: #334 .lr-btn.is-danger:is(:hover, :focus-visible):not(:disabled):not([aria-disabled="true"]) { background: color-mix(in srgb, var(--danger) 10%, var(--paper)); } .lr-btn.is-on { border-color: var(--ink); background: var(--ink); color: var(--paper); } .lr-btn .lr-cap { margin-left: 2px; font-size: 12.5px; font-weight: 600; } +.lr-app-connect { min-width: 40px; height: 40px; } .lr-cap { font-size: 12.5px; font-weight: 600; color: inherit; white-space: nowrap; } /* Sharing and local playback have distinct owners, but one control grammar. */ @@ -1594,14 +1596,11 @@ html[data-theme="dark"] .lr-door-glyph { color: #7d92a8; } .lr-client-access-options { width: 100%; min-width: 0; } .lr-client-access-options > summary { display: flex; align-items: center; justify-content: center; gap: 8px; - width: fit-content; max-width: 100%; min-height: 40px; margin: 0 auto; padding: 8px 12px; - border-radius: 12px; color: var(--muted); font-size: 12px; font-weight: 600; cursor: pointer; - list-style: none; transition: background-color 140ms ease, color 140ms ease; + width: fit-content; max-width: 100%; height: auto; min-height: 40px; margin: 0 auto; padding: 8px 12px; + font-size: 12.5px; font-weight: 600; cursor: pointer; list-style: none; } .lr-client-access-options > summary::-webkit-details-marker { display: none; } -.lr-client-access-options > summary:is(:hover, :focus-visible) { background: var(--control-hover); color: var(--ink); } -.lr-client-access-options > summary:focus-visible { outline: 3px solid var(--action); outline-offset: 2px; } -.lr-client-access-options > summary > svg:last-child { transition: transform 160ms ease; } +.lr-client-access-options > summary > svg:last-child { transition: transform 180ms var(--spring); } .lr-client-access-options[open] > summary > svg:last-child { transform: rotate(180deg); } .lr-client-access-set { display: inline-flex; align-items: center; gap: 4px; color: var(--ink); } .lr-client-access-options .lr-client-access { width: 100%; margin-top: 8px; } @@ -2212,7 +2211,7 @@ html[data-theme="dark"] .lr-tiles .lr-tile:not(.is-selected) small { color: #8ba each stroke. Reduced motion and ordinary controls use the unanimated SVG. */ @media (prefers-reduced-motion: no-preference) { .lr-glyph-draw:not(.lr-spin), - button:is(.lr-btn.is-primary, .lr-tv-big, .lr-join-go, .lr-tv-overlay):not(:disabled):not([aria-disabled="true"]):is(:hover, :focus-visible) .lr-glyph:not(.lr-glyph-draw):not(.lr-spin) { + :is(button, a):is(.lr-btn.is-primary, .lr-tv-big, .lr-join-go, .lr-tv-overlay):not(:disabled):not([aria-disabled="true"]):is(:hover, :focus-visible) .lr-glyph:not(.lr-glyph-draw):not(.lr-spin) { & > :is(path, circle, rect, line) { stroke-dasharray: 1; stroke-dashoffset: 1; diff --git a/src/client/ui/icons.tsx b/src/client/ui/icons.tsx index b53e30a6d..3376ea8ad 100644 --- a/src/client/ui/icons.tsx +++ b/src/client/ui/icons.tsx @@ -121,7 +121,7 @@ export function Glyph({ useEffect(() => { const graphic = graphicRef.current; - if (!draw || !graphic?.closest("button:is(.lr-btn.is-primary, .lr-tv-big, .lr-join-go, .lr-tv-overlay)")) return; + if (!draw || !graphic?.closest(":is(button, a):is(.lr-btn.is-primary, .lr-tv-big, .lr-join-go, .lr-tv-overlay)")) return; return bindSvgReplayOnPointerEnter(graphic); }, [draw]); diff --git a/tests/app-connection-button.test.ts b/tests/app-connection-button.test.ts new file mode 100644 index 000000000..941625f18 --- /dev/null +++ b/tests/app-connection-button.test.ts @@ -0,0 +1,148 @@ +import { readFileSync } from "node:fs"; +import { runInNewContext } from "node:vm"; +import ts from "typescript"; +import { describe, expect, it, vi } from "vitest"; + +// Exercise the button's actual operation and unmount effect without a Browser. +// Discovery has separate transport tests; this covers who may activate a page. +const source = ts.createSourceFile("AppConnectionButton.tsx", readFileSync( + new URL("../src/client/components/living/AppConnectionButton.tsx", import.meta.url), "utf8", +), ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX); +let connect: string | undefined; +let effect: string | undefined; +function collect(node: ts.Node): void { + if (ts.isFunctionDeclaration(node) && node.name?.text === "connect") connect = node.getText(source); + if (ts.isCallExpression(node) && node.expression.getText(source) === "useEffect") { + effect = node.arguments[0]!.getText(source); + } + ts.forEachChild(node, collect); +} +collect(source); +if (!connect || !effect) throw new Error("Missing App connection operation or cleanup"); +const executable = ts.transpileModule(`${connect}\n({ connect, cleanup: (${effect})() });`, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.CommonJS }, +}).outputText; +const settle = () => new Promise(resolve => setImmediate(resolve)); +const site = "https://site.example"; +const approval = `http://127.0.0.1:39721/client?site=${encodeURIComponent(site)}`; + +function mount() { + const popup = { closed: false, close: vi.fn(() => { popup.closed = true; }), location: { replace: vi.fn() } }; + const window = Object.assign(new EventTarget(), { open: vi.fn(() => popup), location: { origin: site } }); + const native = { + discoverNativeConnect: vi.fn(async (_signal: AbortSignal) => approval), + discoverNativeHealth: vi.fn(async (_options: { signal: AbortSignal }): Promise => ({})), + NativeCompatibilityError: class extends Error {}, + }; + const pending = { current: null as { controller: AbortController } | null }; + const setState = vi.fn(); + const activateClient = vi.fn(); + const onConnected = vi.fn(); + const operation = runInNewContext(executable, { + window, pending, setState, activateClient, onConnected, AbortController, URL, + require: (path: string) => { + if (path !== "../../native/client") throw new Error(`Unexpected dependency: ${path}`); + return native; + }, + }) as { connect: () => Promise; cleanup: () => void }; + function message(overrides: Record = {}) { + window.dispatchEvent(Object.assign(new Event("message"), { + source: popup, origin: new URL(approval).origin, data: { type: "piik-app-approved", site }, ...overrides, + })); + } + return { ...operation, popup, window, native, pending, setState, activateClient, onConnected, message }; +} + +describe("App connection approval", () => { + it("requires the exact window, origin and site, then verifies App health once", async () => { + const page = mount(); + let complete!: (health: object) => void; + page.native.discoverNativeHealth.mockImplementation(() => new Promise(resolve => { complete = resolve; })); + await page.connect(); + await page.connect(); + expect(page.window.open).toHaveBeenCalledOnce(); + expect(page.popup.location.replace).toHaveBeenCalledWith(approval); + for (const invalid of [ + { source: {} }, { origin: "http://127.0.0.1:39722" }, + { data: { type: "piik-app-approved", site: "https://another.example" } }, + { data: { type: "other", site } }, { data: null }, + ]) page.message(invalid); + expect(page.native.discoverNativeHealth).not.toHaveBeenCalled(); + page.message(); + page.message(); + expect(page.native.discoverNativeHealth).toHaveBeenCalledOnce(); + expect(page.activateClient).not.toHaveBeenCalled(); + complete({}); + await settle(); + expect(page.activateClient).toHaveBeenCalledOnce(); + expect(page.onConnected).toHaveBeenCalledOnce(); + expect(page.popup.closed).toBe(true); + expect(page.pending.current).toBeNull(); + expect(page.native.discoverNativeConnect.mock.calls[0]![0].aborted).toBe(true); + page.message(); + expect(page.native.discoverNativeHealth).toHaveBeenCalledOnce(); + }); + + it.each(["absent", "failed"])("does not activate from an approval message when health is %s", async result => { + const page = mount(); + if (result === "absent") page.native.discoverNativeHealth.mockResolvedValue(null); + else page.native.discoverNativeHealth.mockRejectedValue(new Error("network unavailable")); + await page.connect(); + page.message(); + await settle(); + expect(page.activateClient).not.toHaveBeenCalled(); + expect(page.onConnected).not.toHaveBeenCalled(); + expect(page.setState).toHaveBeenLastCalledWith("unavailable"); + expect(page.popup.closed).toBe(true); + expect(page.pending.current).toBeNull(); + await page.connect(); + expect(page.window.open).toHaveBeenCalledTimes(2); + page.cleanup(); + }); + + it("ignores a late healthy response after unmount and closes the popup", async () => { + const page = mount(); + let complete!: (health: object) => void; + page.native.discoverNativeHealth.mockImplementation(() => new Promise(resolve => { complete = resolve; })); + await page.connect(); + page.message(); + page.cleanup(); + complete({}); + await settle(); + page.message(); + expect(page.native.discoverNativeHealth).toHaveBeenCalledOnce(); + expect(page.activateClient).not.toHaveBeenCalled(); + expect(page.onConnected).not.toHaveBeenCalled(); + expect(page.popup.closed).toBe(true); + expect(page.native.discoverNativeHealth.mock.calls[0]![0].signal.aborted).toBe(true); + expect(page.setState).toHaveBeenLastCalledWith("busy"); + }); + + it("never navigates the popup when discovery completes after unmount", async () => { + const page = mount(); + let complete!: (url: string) => void; + page.native.discoverNativeConnect.mockImplementation(() => new Promise(resolve => { complete = resolve; })); + const connecting = page.connect(); + await settle(); + page.cleanup(); + complete(approval); + await connecting; + expect(page.popup.location.replace).not.toHaveBeenCalled(); + expect(page.popup.closed).toBe(true); + expect(page.activateClient).not.toHaveBeenCalled(); + }); + + it("releases a closed approval window on focus and permits retry", async () => { + const page = mount(); + await page.connect(); + page.popup.closed = true; + page.window.dispatchEvent(new Event("focus")); + expect(page.pending.current).toBeNull(); + expect(page.setState).toHaveBeenLastCalledWith("idle"); + page.message(); + expect(page.native.discoverNativeHealth).not.toHaveBeenCalled(); + await page.connect(); + expect(page.window.open).toHaveBeenCalledTimes(2); + page.cleanup(); + }); +}); From dc4939ad4536cc888baffda0a3711e02be992093 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Mon, 12 Oct 2026 01:04:32 +0800 Subject: [PATCH 171/262] fix(host): retire native selection with room authority --- src/client/pages/HostPage.tsx | 68 +++++--------- tests/host-quality-ownership.test.ts | 133 +++++++++++++++++++++++++-- 2 files changed, 152 insertions(+), 49 deletions(-) diff --git a/src/client/pages/HostPage.tsx b/src/client/pages/HostPage.tsx index c7f1978a0..903df0d34 100644 --- a/src/client/pages/HostPage.tsx +++ b/src/client/pages/HostPage.tsx @@ -900,6 +900,7 @@ export function HostPage({ } function disposeResources(notifyServer: boolean, keepRoomSession = false): void { + closeCaptureSourcePicker(); sourceSwitchRef.current = null; qualityChangeRef.current = null; pendingQualityChangeRef.current = null; @@ -1023,7 +1024,6 @@ export function HostPage({ } activeGenerationRef.current = null; generationRef.current += 1; - closeCaptureSourcePicker(); const currentRoom = roomRef.current; if (notifyServer && currentRoom) { writePreferredRoom(currentRoom.roomId); @@ -1033,6 +1033,21 @@ export function HostPage({ setPhase("ended"); } + function endRoomSession( + message: StatusNoticeMessage, + comic: ComicKind, + tone: ComicTone = "bad", + idlePhase: "error" | "ended" = "error", + ): void { + if (activeGenerationRef.current !== null) { + endSharing(message, false, comic, tone, false); + } else { + disposeResources(false); + setStatusNotice(message, comic, tone); + setPhase(idlePhase); + } + } + async function replaceCurrentRoom(): Promise { const activeRoom = roomRef.current; if (!activeRoom || roomMutationRef.current || phase === "starting") { @@ -1120,9 +1135,7 @@ export function HostPage({ } } if (forgetRoom(activeRoom)) { - // endSharing already stated the recovery path; the raw room error - // would only overwrite it with the same 404 in server wording. - endSharing({ key: "host.roomInvalid" }, false, "room-not-found", "bad", false); + endRoomSession({ key: "host.roomInvalid" }, "room-not-found"); return; } } @@ -2532,24 +2545,12 @@ export function HostPage({ }, onTerminated: (reason) => { if (signalRef.current !== signal || !isCurrentRoomAuthority(activeRoom)) return; - if (activeGenerationRef.current !== null) { - endSharing({ key: hostTerminationKey(reason) }, false, "signal-failed", "bad", false); - } else { - disposeResources(false); - setStatusNotice({ key: hostTerminationKey(reason) }, "signal-failed", "bad"); - setPhase("error"); - } + endRoomSession({ key: hostTerminationKey(reason) }, "signal-failed"); if (reason === "SESSION_REPLACED") forgetRoom(activeRoom, true); }, onAccessRequired: () => { if (signalRef.current !== signal || !isCurrentRoomAuthority(activeRoom)) return; - if (activeGenerationRef.current !== null) { - endSharing({ key: "gate.expired" }, false, "access-denied", "bad", false); - } else { - disposeResources(false); - setStatusNotice({ key: "gate.expired" }, "access-denied", "bad"); - setPhase("error"); - } + endRoomSession({ key: "gate.expired" }, "access-denied"); onAuthorizationRequired?.(); }, onMessage: (message) => { @@ -2573,8 +2574,7 @@ export function HostPage({ // Capture may still be awaiting permission. Its ready path owns // creation/publication; room recovery must not start it early. if (!localShareActive) { - setStatusNotice({ key: "host.roomInvalid" }, "room-not-found", "bad"); - setPhase("error"); + endRoomSession({ key: "host.roomInvalid" }, "room-not-found"); } return; } @@ -2586,35 +2586,16 @@ export function HostPage({ } if (message.type === "error" && message.code === "INVALID_TOKEN") { forgetRoom(activeRoom); - if (localShareActive) { - endSharing({ key: "host.roomInvalid" }, false, "room-not-found", "bad", false); - } else { - disposeResources(false); - forgetRoom(activeRoom); - setStatusNotice({ key: "host.roomInvalid" }, "room-not-found", "bad"); - setPhase("error"); - } + endRoomSession({ key: "host.roomInvalid" }, "room-not-found"); return; } if (message.type === "room-closed") { if (!forgetRoom(activeRoom)) return; - if (localShareActive) { - endSharing({ key: "host.roomClosed" }, false, "room-closed", "off", false); - } else { - disposeResources(false); - setStatusNotice({ key: "host.roomClosed" }, "room-closed", "off"); - setPhase("ended"); - } + endRoomSession({ key: "host.roomClosed" }, "room-closed", "off", "ended"); return; } if (message.type === "error" && message.code === "HOST_ALREADY_CONNECTED") { - if (localShareActive) { - endSharing(hostServerErrorNotice(message.code), false, "signal-failed", "bad", false); - } else { - disposeResources(false); - setStatusNotice(hostServerErrorNotice(message.code), "signal-failed", "bad"); - setPhase("error"); - } + endRoomSession(hostServerErrorNotice(message.code), "signal-failed"); return; } const reauthenticated = @@ -3285,7 +3266,8 @@ export function HostPage({ onAuthorizationRequired?.(); } if (error instanceof ApiError && error.status === 404) { - forgetRoom(activeRoom); + if (forgetRoom(activeRoom)) endRoomSession({ key: "host.roomInvalid" }, "room-not-found"); + return; } if (!isCurrentRoomAuthority(activeRoom) && roomRef.current !== null) { return; diff --git a/tests/host-quality-ownership.test.ts b/tests/host-quality-ownership.test.ts index f925485b9..986ec250c 100644 --- a/tests/host-quality-ownership.test.ts +++ b/tests/host-quality-ownership.test.ts @@ -13,6 +13,7 @@ import { nativeCaptureTargetKey } from "../src/client/native/capture-selection"; import { NativeMediaBridgeError } from "../src/client/native/media-bridge"; import { reconcileBoundedMediaChildren } from "../src/client/webrtc/media-assignment"; import { debugError, debugEvent, debugOperation } from "../src/client/lib/debug"; +import { ApiError } from "../src/client/lib/api"; import { hostActionErrorNotice, hostFailureCode, isCapturePermissionFailure } from "../src/client/pages/host-page-notices"; // Exercise the actual page owners without mounting capture hardware or a Browser. @@ -25,7 +26,8 @@ const owners = new Set(["changeQuality", "commitQuality", "rememberQuality", "lo "ownNativeClient", "discardNativeClient", "releaseUnusedNativeClient", "closeCaptureSourcePicker", "openCaptureSourcePicker", "startBrowserShareFromPicker", "startNativeShareFromPicker", "startSharing", "connectRoomSignal", "createReplacementRoom", "beginRoomMutation", "finishRoomMutation", "setCaptureError", "changeMicrophone", "toggleSharingPause", - "startPeer", "reconcileHostChildren", "setNotice", "setNoticeKey", "setStatusNotice", "setNoticeError", "setNoticeErrorKey", "endSharing", "hostTerminationKey", "copyRoomLink", "isCurrentRoomAuthority"]); + "startPeer", "reconcileHostChildren", "setNotice", "setNoticeKey", "setStatusNotice", "setNoticeError", "setNoticeErrorKey", "endSharing", "disposeResources", "hostTerminationKey", "copyRoomLink", "isCurrentRoomAuthority", + "handleRoomAccessFailure", "replaceCurrentRoom", "forgetRoom", "endRoomSession"]); const functions: string[] = []; function collect(node: ts.Node): void { @@ -87,7 +89,7 @@ function fixture(launchedByClient = true) { setCopiedRoomLink: vi.fn(), setPhase: vi.fn(), roomInitializationRef: ref(Promise.resolve()), roomRef: ref(null), createRoom: vi.fn(async () => { throw new Error("must not create an empty room"); }), - readPreferredRoomId: () => null, disposeResources: vi.fn(), ApiError: class extends Error {}, + readPreferredRoomId: () => null, disposeResources: vi.fn(), ApiError, NativeMediaBridge: vi.fn(function (_shareId: string, _client: unknown, _onFailed: () => void) { return bridge; }), manualVideoCodecPreference: vi.fn(), phase: "live", qualitySettingsRef: ref(original), advancedQualityRef: ref(original), qualityChangeRef: ref(null), pendingQualityChangeRef: ref(null), @@ -130,14 +132,16 @@ function fixture(launchedByClient = true) { const start = context.startSharing; const startChild = context.startPeer; const openPicker = context.openCaptureSourcePicker; + const dispose = context.disposeResources as (notifyServer: boolean, keepRoomSession?: boolean) => void; const writeNoticeError = context.setNoticeError; state.setNoticeError.mockImplementation(writeNoticeError); context.setNoticeError = state.setNoticeError; context.startSharing = state.startSharing; context.startPeer = state.startPeer; context.openCaptureSourcePicker = state.openCaptureSourcePicker; + context.disposeResources = state.disposeResources; return { ...state, peer, client, route, track, bridge, physical: () => physical, - context, startChild, openPicker: openPicker as () => Promise, + context, startChild, dispose, openPicker: openPicker as () => Promise, closePicker: context.closeCaptureSourcePicker as () => void, releaseUnused: context.releaseUnusedNativeClient as () => void, disposeNative: context.disposeNativeShare as (expectedShare?: string) => void, @@ -155,6 +159,22 @@ function fixture(launchedByClient = true) { }; } +function useRealResourceDisposal(current: ReturnType): void { + current.peersRef.current.clear(); + Object.assign(current.hostSfuRouteRef, { current: null }); + Object.assign(current.context, { + codecProbeAbortRef: ref(null), VP8_ONLY_VIDEO_CODEC: { primary: "vp8" }, + setResolvedVideoCodec: vi.fn(), setHostSfuQualityWarning: vi.fn(), + setInteractionSession: vi.fn(), browserVideoPoolRef: ref(null), + setMicrophonePending: vi.fn(), retiringStreamRef: ref(null), iceConfigRef: ref(null), + setStream: vi.fn(), setPeerSnapshots: vi.fn(), setParticipantPresence: vi.fn(), + viewerQualityEvidenceStore: { clear: vi.fn(), getSnapshot: () => new Map() }, + cancelViewerQualityEvidenceRender: vi.fn(), setViewerQualityEvidence: vi.fn(), + setSignalStatus: vi.fn(), sharingPausedRef: ref(false), setSharingPaused: vi.fn(), + }); + current.disposeResources.mockImplementation(current.dispose); +} + describe("Camera replacement ownership", () => { it("keeps a screen Stop authoritative while a camera picker is pending", async () => { const current = fixture(); @@ -211,6 +231,7 @@ describe("Host room ownership", () => { class Signal { interactions = {}; stop = vi.fn(); + stopSharing = vi.fn(); constructor(public identity: { roomId: string; token: string; roomOnly?: true; shareGeneration?: string }, public events: { onMessage(message: unknown): void; @@ -220,7 +241,7 @@ describe("Host room ownership", () => { ownsHostRoom(roomId: string, token: string) { return this.identity.roomId === roomId && this.identity.token === token; } wantsHostPublication(generation: string) { return this.identity.shareGeneration === generation && this.identity.roomOnly !== true; } } - const forgetRoom = vi.fn(() => { Object.assign(current.roomRef, { current: null }); return true; }); + const forgetRoom = vi.fn(current.context.forgetRoom); Object.assign(current.roomRef, { current: room }); Object.assign(current.signalRef, { current: null }); current.generationRef.current = 1; @@ -230,7 +251,7 @@ describe("Host room ownership", () => { getStableClientId: () => "host-client", defaultHostDisplayName: () => "Host", readDisplayName: () => "Host", setDisplayName: vi.fn(), setDisplayNameDraft: vi.fn(), setDisplayNameError: vi.fn(), setInteractionSession: vi.fn(), setSignalStatus: vi.fn(), forgetRoom, - onAuthorizationRequired: vi.fn(), + onAuthorizationRequired: vi.fn(), clearHostRoom: vi.fn(), writePreferredRoom: vi.fn(), hostServerErrorNotice: (code: string) => code, }); return { ...current, room, signals, forgetRoom, connect: (value = room) => current.context.connectRoomSignal(value) as Signal }; @@ -270,6 +291,105 @@ describe("Host room ownership", () => { }); }); + it.each(["INVALID_TOKEN", "HOST_ALREADY_CONNECTED", "room-closed", "terminated", "access-required"])( + "retires idle native selection and its late results on %s", async event => { + for (const stage of ["ready", "discovery", "enumeration"]) { + const current = roomFixture(); + useRealResourceDisposal(current); + current.context.phase = "idle"; + current.activeGenerationRef.current = null; + current.shareGenerationRef.current = null; + current.nativeShareGenerationRef.current = null; + const discovery = deferred(); + const enumeration = deferred(); + if (stage === "discovery") { + current.nativeClientRef.current = null; + current.NativeClient.connect.mockReturnValueOnce(discovery.promise); + } else if (stage === "enumeration") current.client.sources.mockReturnValueOnce(enumeration.promise); + const signal = current.connect(); + const selecting = current.openPicker(); + if (stage === "ready") await selecting; + else await vi.waitFor(() => expect(stage === "discovery" + ? current.NativeClient.connect : current.client.sources).toHaveBeenCalledOnce()); + expect(current.nativeSourceRequestRef.current).not.toBeNull(); + + if (event === "terminated") signal.events.onTerminated("SIGNAL_TERMINATED"); + else if (event === "access-required") signal.events.onAccessRequired(); + else signal.events.onMessage(event === "room-closed" ? { type: event } : { type: "error", code: event }); + expect(current.nativeSourceRequestRef.current).toBeNull(); + expect(current.nativeSourcePathRef.current).toBeNull(); + expect(current.nativeClientConnectRef.current).toBeNull(); + discovery.resolve(current.client); + enumeration.resolve([]); + await selecting; + expect(current.nativeClientRef.current).toBeNull(); + expect(current.client.close).toHaveBeenCalledOnce(); + expect(current.setNativeSources).toHaveBeenLastCalledWith(null); + expect(current.createRoom).not.toHaveBeenCalled(); + if (stage === "discovery") expect(current.client.sources).not.toHaveBeenCalled(); + } + }, + ); + + it.each([false, true])("retires a room rejected by an access update (sharing=%s)", async sharing => { + const current = roomFixture(); + useRealResourceDisposal(current); + current.context.phase = sharing ? "live" : "idle"; + current.nativeModeRef.current = sharing; + if (!sharing) { + current.activeGenerationRef.current = null; + current.shareGenerationRef.current = null; + current.nativeShareGenerationRef.current = null; + current.streamRef.current = null; + } + const signal = current.connect(); + await current.openPicker(); + current.context.handleRoomAccessFailure(new ApiError("Room not found", 404), current.room); + expect(current.roomRef.current).toBeNull(); + expect(current.signalRef.current).toBeNull(); + expect(signal.stop).toHaveBeenCalledOnce(); + expect(current.activeGenerationRef.current).toBeNull(); + expect(current.track.stop).toHaveBeenCalledTimes(sharing ? 1 : 0); + expect(current.client.close).toHaveBeenCalledOnce(); + expect(current.setNativeSources).toHaveBeenLastCalledWith(null); + expect(current.setNoticeValue).toHaveBeenLastCalledWith(expect.objectContaining({ + key: "host.roomInvalid", target: "television", comic: "room-not-found", tone: "bad", + })); + }); + + it("retires the room session when a replacement fails after capture was stopped", async () => { + const current = roomFixture(); + useRealResourceDisposal(current); + const response = deferred(); + current.context.replaceOwnedRoom = vi.fn(() => response.promise); + const signal = current.connect(); + const replacing = current.context.replaceCurrentRoom(); + current.context.endSharing({ key: "host.stopNotice" }); + expect(signal.stop).not.toHaveBeenCalled(); + expect(current.signalRef.current).toBe(signal); + response.reject(new ApiError("Room not found", 404)); + await replacing; + expect(current.roomRef.current).toBeNull(); + expect(current.signalRef.current).toBeNull(); + expect(signal.stop).toHaveBeenCalledOnce(); + expect(current.createRoom).not.toHaveBeenCalled(); + expect(current.roomMutationRef.current).toBeNull(); + }); + + it("ignores an old room's failed access update after another room owns the session", () => { + const current = roomFixture(); + useRealResourceDisposal(current); + const replacement = { ...current.room, hostToken: "replacement-token" }; + Object.assign(current.roomRef, { current: replacement }); + const signal = current.connect(replacement); + current.context.handleRoomAccessFailure(new ApiError("Room not found", 404), current.room); + expect(current.roomRef.current).toBe(replacement); + expect(current.signalRef.current).toBe(signal); + expect(signal.stop).not.toHaveBeenCalled(); + expect(current.client.close).not.toHaveBeenCalled(); + expect(current.setNoticeValue).not.toHaveBeenCalled(); + }); + it.each(["INVALID_TOKEN", "HOST_ALREADY_CONNECTED", "room-closed", "terminated", "access-required"])( "keeps %s in the same status surface with and without a source", event => { const outcomes: unknown[] = []; @@ -643,7 +763,8 @@ describe("Host quality ownership", () => { current.nativeModeRef.current = true; await current.openPicker(); expect(current.nativeSourceRequestRef.current).not.toBeNull(); - current.disposeResources.mockImplementation(() => current.disposeNative()); + useRealResourceDisposal(current); + Object.assign(current.signalRef, { current: null }); current.context.endSharing({ key: "host.notice.stopped" }); expect(current.nativeSourceRequestRef.current).toBeNull(); expect(current.nativeSourcePathRef.current).toBeNull(); From 7f40cec61b70530ad506c12f5489625af0857281 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Mon, 12 Oct 2026 01:09:03 +0800 Subject: [PATCH 172/262] fix(ci): keep candidate builds independent of release history --- .github/workflows/repository-hygiene.yml | 2 +- docs/standards/versioning.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/repository-hygiene.yml b/.github/workflows/repository-hygiene.yml index 708e5c47c..1831958e5 100644 --- a/.github/workflows/repository-hygiene.yml +++ b/.github/workflows/repository-hygiene.yml @@ -75,7 +75,7 @@ jobs: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && vars.PIIK_RELEASES_ENABLED == 'true' && 'plan' || 'build' }} --github-output - name: Preview release notes - if: steps.release.outputs.publish == 'true' || inputs.candidate_version != '' + if: steps.release.outputs.publish == 'true' run: node scripts/release-notes.mjs "$PIIK_RELEASE_VERSION" "$GITHUB_SHA" --github-summary env: PIIK_RELEASE_VERSION: ${{ steps.release.outputs.version }} diff --git a/docs/standards/versioning.md b/docs/standards/versioning.md index 34650a258..42ae5e1e5 100644 --- a/docs/standards/versioning.md +++ b/docs/standards/versioning.md @@ -237,11 +237,11 @@ first-parent commits since the previous stable tag. Standalone website and documentation phases need no product release notes. The first release uses only its launch commit's product introduction; private development history is not a launch changelog. Missing, empty, duplicate or visibly encoding-damaged -sections fail validation before packaging/publication. GitHub Actions shows the generated text in its -run summary. A manual candidate with an explicit version also previews it, so -prepare its commit text before dispatching a release rehearsal. A manual -candidate with no product changes may still be built; its summary reports that -there are no product release notes, and publication remains disabled. +sections fail PR and publishing validation. GitHub Actions shows the generated +release text in its publishing run summary. Manual candidates validate and build +the requested revision without requiring release copy in branch checkpoints; +their version labels artifacts and never enables publication. The PR body remains +the review surface until integration preserves it in the squash commit. Supply multiline PR/release copy through a UTF-8 file and read back the stored body before integration. A terminal preview alone does not prove that Chinese From 69211f9c4297aad4a1678ce2626c07e883e01f02 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Mon, 12 Oct 2026 01:24:07 +0800 Subject: [PATCH 173/262] docs: focus candidate ledger on remaining acceptance --- docs/todo.md | 79 +++++++++++++++++----------------------------------- 1 file changed, 26 insertions(+), 53 deletions(-) diff --git a/docs/todo.md b/docs/todo.md index 17b90c791..116d23dc7 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -1,6 +1,6 @@ # Current TODO Ledger -Last reviewed: 2026-10-11 +Last reviewed: 2026-10-12 The current 2.0 preparation includes independent investigation across this ledger. Explicit owner holds remain in force; an investigation does not itself @@ -13,9 +13,10 @@ Git/PRs own completed history. - [ ] **2.0 candidate preparation and acceptance.** The owner reopened investigation, useful improvements and outstanding acceptance on 2026-10-07, and on 2026-10-09 chose to hold the maintenance changes for 2.0 rather than publish them separately. - Prepare one coherent candidate and stop before publication. Preserve the existing - product model; evaluate held features against measurable benefit and total complexity - before accepting a changed contract. + Complete final packaged lifecycle and device/network acceptance, then present + the frozen candidate and its bilingual release notes for owner review before + publication. Retain explicit 1.x mismatch checks. The retired Magicsock package + is not the current candidate; preserve the existing product model and held scope. - [ ] **Linux Portal target identity acceptance (#470).** The [contributor fix](https://github.com/TNTcraftHIM/Piik/pull/470) is integrated with authorship retained: serial and node-ID targeting remain distinct. @@ -27,10 +28,10 @@ Git/PRs own completed history. [#469](https://github.com/TNTcraftHIM/Piik/pull/469)'s 4K request overlap the numeric-media candidate; neither PR has been merged. 4K60 is included in candidate admission, alongside the tested wider video control range, and still - needs physical device/network acceptance. Preserve - contributor credit and explain the disposition at integration. Rewrite - [#459](https://github.com/TNTcraftHIM/Piik/pull/459)'s earlier maintenance scope - and compatibility claims around the final 2.0 result before integration. + needs physical device/network acceptance. Preserve contributor credit and + explain the disposition at integration. Keep + [#459](https://github.com/TNTcraftHIM/Piik/pull/459)'s candidate scope and + bilingual upgrade instructions aligned with the accepted result. - [ ] **Independent Gitee mirror recovery.** Implementation and local acceptance are complete. Integrate through the next owner-approved publishing merge and verify the separate workflow with its original CI artifacts. Mirror failure @@ -68,46 +69,19 @@ Existing negative results narrow the next experiment; they do not freeze the whole candidate or require another version-number decision before other work. Implementation needs a concrete useful result within the product model. -- [ ] **Media parameters and capabilities for 2.0.** Follow the - [media assessment](./research/native-client-media.md#media-capability-extension-assessment): - separate UI presets, numeric requests, bounded implementation capabilities and - actual output. Numeric request, capability and observation boundaries are now - separate from implementation admission. Keep receiver, codec/alignment and - resource budgets intact; opening an output range still needs end-to-end evidence. - The inherited menu-only source range is replaced by 1..60 fps / 0.1..50 Mbps, - with coherent lower-output bounds and actual-progress preflight. Local - production-codec, source-change, Browser loopback, Native Host and SFU checks - cover both rate bounds. Preserve these checks in final package acceptance. - Keep the audio sender ceiling backed by the existing packet-size evidence. - Numeric audio and video controls are implemented; the assessment - owns local capture, codec, derived-output, SFU and bounded mixed-route evidence. - Remaining receiver acceptance needs physical limited decoders and camera - orientation behavior, beyond the bounded SDP-limit and mixed-route controls. - Browser display capture, Browser relay, Browser-to-Native fanout and SFU have - bounded high-resolution continuity evidence. macOS geometry hardware - checks and physical 4K/game capture remain open; both macOS package targets - now pass SDK compilation and startup. The fanout controls did not - establish sustained 60 fps; retain that performance limit when arranging - device/network acceptance. Preserve safety limits and - framework-owned adaptation. The - [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment) - retain explicit receive-limit enforcement and the current 320 kbps sender - ceiling. Physical listening, mobile and field-network acceptance remain open; - research owns the bounded real-Opus impairment and video-recovery control. - Validate the existing mono/stereo and Windows HDR-to-SDR paths under the - [fidelity boundary](./research/media-fidelity.md); surround delivery and - end-to-end HDR are outside this candidate. Additional codecs are also outside - this candidate. Later inclusion needs complete - encode/decode/forward support and a late-Viewer policy; - selecting a codec cannot transcode an existing publication. Final package - acceptance must retain the verified live-change, relay/SFU and limited-receiver - behavior, and check explicit 1.x mismatch with packaged builds. The published - 1.x enums remain that major's baseline; the PR is not a compatible enum-only extension - of v1.9.2. -- [ ] **2.0 candidate acceptance.** Finish the combined numeric-media UI, - App/room lifecycle, ordinary Browser/Native routes and regression checks before - asking the owner to arrange external testing. Keep each test build frozen and - identifiable; the retired Magicsock package is not the current candidate. +- [ ] **Numeric-media device acceptance.** The implemented candidate and bounded + local evidence are owned by the + [media assessment](./research/native-client-media.md#media-capability-extension-assessment). + Retain live changes, source replacement, relay/SFU, limited-receiver and + low/high-rate controls in final package acceptance. Remaining physical checks + include limited decoders, camera orientation, macOS geometry and 4K/game capture; + local fanout controls do not establish sustained physical 4K60. Complete physical + listening, mobile and field-network checks for the + [audio controls](./research/browser-screen-audio-quality.md#numeric-bitrate-assessment), + preserving their packet and receive limits. Validate existing mono/stereo and + Windows HDR-to-SDR under the [fidelity boundary](./research/media-fidelity.md). + Additional codecs, surround delivery and end-to-end HDR remain outside this + candidate; product boundaries belong to the media-quality owner. - [ ] **Managed Demo hosting.** The [provider assessment](./operations/website.md#managed-hosting-assessment) identifies UDP/storage requirements but establishes no cost saving. Keep the existing P2P-only Demo until a provider and operating boundary are accepted. @@ -116,11 +90,10 @@ Implementation needs a concrete useful result within the product model. existing device selector. Keep it outside this candidate until a concrete use case justifies that boundary and physical latency/echo/background checks. This proposal does not reopen room voice. -- [ ] **Passive App attachment acceptance.** The two-choice launcher and explicit - local Site approval replace the third mode, using the existing native runtime. - Include first connection, saved approval, replacement/revocation and Local - coexistence in final packaged acceptance. Browser permission and cross-platform - checks must not be inferred from the local development preview. +- [ ] **Passive App attachment acceptance.** Preserve the verified first + connection, saved approval, replacement/revocation and Local coexistence in + final packaged acceptance. Check Browser permission and cross-platform behavior; + local approval controls do not establish those physical-device outcomes. ## Awaiting Device Or Reporter Evidence From ac970ce1cfc9ab6d82e0a60840a2d2ec42d719ff Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:44:08 +0800 Subject: [PATCH 174/262] fix(ui): apply quality choices without committing superseded drafts --- .../components/living/QualityPresets.tsx | 86 +++++++++++-------- src/client/components/living/primitives.tsx | 9 +- tests/control-activation.test.ts | 41 +++++++++ tests/quality-controls.test.ts | 81 +++++++++++++++++ 4 files changed, 176 insertions(+), 41 deletions(-) create mode 100644 tests/control-activation.test.ts create mode 100644 tests/quality-controls.test.ts diff --git a/src/client/components/living/QualityPresets.tsx b/src/client/components/living/QualityPresets.tsx index 92679de4c..ea1cdc1b2 100644 --- a/src/client/components/living/QualityPresets.tsx +++ b/src/client/components/living/QualityPresets.tsx @@ -1,4 +1,4 @@ -import { Fragment, useRef } from "react"; +import { Fragment, useRef, type ReactNode } from "react"; import { MIN_AUDIO_BITRATE } from "../../../shared/protocol"; import { QUALITY_PROFILES, QUALITY_PROFILE_KEYS, QUALITY_RESOLUTIONS, qualityResolutionEqual, @@ -108,7 +108,7 @@ const AUDIO_QUALITY_CAPTIONS: Record = { "very-high": "host.advanced.audio.veryHigh", }; -function QualityNumberInput({ value, min, max, step, unit, name, label, disabled, onChange }: { +function QualityNumberControl({ value, min, max, step, unit, name, label, disabled, onChange, className, children }: { value: number; min: number; max: number; @@ -118,27 +118,34 @@ function QualityNumberInput({ value, min, max, step, unit, name, label, disabled label: string; disabled: boolean; onChange: (value: number) => void; + className: string; + children: (input: ReactNode, select: (value: number) => void) => ReactNode; }) { - return