diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 90002d9552..7d197dbdbd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -92,6 +92,12 @@ jobs: if: ${{ !cancelled() }} run: nix develop --command bun js/net/bench/frames.ts + # Times one varint encode and decode per wire format. No threshold: it only + # has to keep running, and its numbers are the baseline for the generated codec. + - name: JS varint benchmark + if: ${{ !cancelled() }} + run: nix develop --command bun js/net/bench/varint.ts + # Fails if publishing a group costs more as the track retains more groups, # which means the latency guard or the cache went back to scanning them all. - name: JS track retention benchmark diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 180c814f0a..fdc5cd3c99 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -1652,3 +1652,8 @@ for (const end of [ } }); } + +test("LegacyFormat rejects a timestamp past 2^53 - 1 instead of rounding", () => { + const frame = Varint.encode(2n ** 53n + 1n); + expect(() => new LegacyFormat("video").decode(frame)).toThrow(/larger than 53-bits/); +}); diff --git a/js/loc/src/index.test.ts b/js/loc/src/index.test.ts index c9d0b1f27a..a3f41232e3 100644 --- a/js/loc/src/index.test.ts +++ b/js/loc/src/index.test.ts @@ -133,3 +133,19 @@ test("Format decodes the draft-03 timestamp property", () => { expect(decoded.timestamp).toBe(4242 as Time.Micro); expect(decoded.payload).toEqual(payload); }); + +test("Format skips an unknown property whose value needs all 62 bits", () => { + const props = concat( + Varint.encode(0x02), + Varint.encode(2n ** 62n - 1n), + Varint.encode(PROP_TIMESTAMP - 0x02), + Varint.encode(1_000), + ); + const [decoded] = new Format().decode(buildFrame(props, new Uint8Array([1]))); + expect(decoded.timestamp).toBe(1_000 as Time.Micro); +}); + +test("Format rejects a timestamp past 2^53 - 1 instead of rounding", () => { + const props = concat(Varint.encode(PROP_TIMESTAMP), Varint.encode(2n ** 53n)); + expect(() => new Format().decode(buildFrame(props, new Uint8Array()))).toThrow(/larger than 53-bits/); +}); diff --git a/js/loc/src/index.ts b/js/loc/src/index.ts index 57c295915c..faafef2905 100644 --- a/js/loc/src/index.ts +++ b/js/loc/src/index.ts @@ -73,17 +73,18 @@ export class Format { prevType = abs; cursor = afterDelta; - if (abs % 2 === 0) { - const [value, afterValue] = Moq.Varint.decode(cursor); - cursor = afterValue; - if (abs === PROP_TIMESTAMP || abs === PROP_TIMESTAMP_DRAFT03) { - timestamp = value; - } else if (abs === PROP_TIMESCALE) { - if (value === 0) { - throw new Error("loc: timescale property must be non-zero"); - } - timescale = value; + if (abs === PROP_TIMESTAMP || abs === PROP_TIMESTAMP_DRAFT03) { + [timestamp, cursor] = Moq.Varint.decode(cursor); + } else if (abs === PROP_TIMESCALE) { + let value: number; + [value, cursor] = Moq.Varint.decode(cursor); + if (value === 0) { + throw new Error("loc: timescale property must be non-zero"); } + timescale = value; + } else if (abs % 2 === 0) { + // An unknown varint property may use all 62 bits, which a number can't hold; skip it. + cursor = Moq.Varint.decodeBigInt(cursor)[1]; } else { const [len, afterLenInner] = Moq.Varint.decode(cursor); if (afterLenInner.byteLength < len) { diff --git a/js/net/bench/varint.ts b/js/net/bench/varint.ts new file mode 100644 index 0000000000..d8322a1c1e --- /dev/null +++ b/js/net/bench/varint.ts @@ -0,0 +1,89 @@ +/** Time one varint encode and one decode, for both wire formats, at their 1, 2, 4, and 8-byte sizes. */ +import { Version } from "../src/ietf/version.ts"; +import { Cursor } from "../src/stream.ts"; +import * as Varint from "../src/varint.ts"; + +// Decodes run through one Cursor over this many copies, so the Cursor itself isn't timed. +const run = 1024; +const runs = 2_000; +const reps = 9; +const scratch = new ArrayBuffer(9); +let checksum = 0; + +// QUIC varints (lite, and moq-transport before draft-17), and leading-ones (draft-17+), each at the +// largest value of its 1, 2, 4, and 8-byte forms that a `number` holds. +const formats = [ + { + name: "quic", + version: undefined, + encode: Varint.encodeTo, + values: [2 ** 6 - 1, 2 ** 14 - 1, 2 ** 30 - 1, Number.MAX_SAFE_INTEGER], + }, + { + name: "leading-ones", + version: Version.DRAFT_17, + encode: Varint.encodeLeadingOnesTo, + values: [2 ** 7 - 1, 2 ** 14 - 1, 2 ** 28 - 1, Number.MAX_SAFE_INTEGER], + }, +]; + +interface Case { + name: string; + ops: [string, () => void][]; +} + +const cases: Case[] = []; +for (const format of formats) { + for (const value of format.values) { + const one = format.encode(scratch, value).slice(); + const encoded = new Uint8Array(one.byteLength * run); + for (let i = 0; i < run; i++) encoded.set(one, i * one.byteLength); + + cases.push({ + name: `${format.name},${one.byteLength}-byte`, + ops: [ + [ + "encode", + () => { + for (let i = 0; i < run; i++) checksum += format.encode(scratch, value).byteLength; + }, + ], + [ + "decode", + () => { + const cursor = new Cursor(encoded, format.version); + for (let i = 0; i < run; i++) checksum += cursor.u53(); + }, + ], + // The same, as the U64 the generated codec will use, which allocates. + [ + "decode-varint", + () => { + const cursor = new Cursor(encoded, format.version); + for (let i = 0; i < run; i++) checksum += cursor.varint().lo; + }, + ], + ], + }); + } +} + +// Nanoseconds per varint for the fastest of several reps, since a slower one measured the machine. +function time(body: () => void): number { + let best = Number.POSITIVE_INFINITY; + for (let rep = 0; rep < reps; rep++) { + const start = performance.now(); + for (let i = 0; i < runs; i++) body(); + best = Math.min(best, ((performance.now() - start) * 1e6) / (runs * run)); + } + return best; +} + +// Run every case once first, so the JIT has seen all of them and the first row isn't timing warmup. +for (const { ops } of cases) for (const [, body] of ops) for (let i = 0; i < runs; i++) body(); + +console.log("format,value,op,ns_per_op"); +for (const { name, ops } of cases) { + for (const [op, body] of ops) console.log(`${name},${op},${time(body).toFixed(1)}`); +} +if (checksum === 0) throw new Error("benchmark did no work"); diff --git a/js/net/src/index.ts b/js/net/src/index.ts index 932f7adc23..7958c42d52 100644 --- a/js/net/src/index.ts +++ b/js/net/src/index.ts @@ -28,5 +28,5 @@ export * as Path from "./path.ts"; export * as Time from "./time.ts"; /** Track role handles. */ export * as Track from "./track.ts"; -/** QUIC variable-length integer encoding and decoding. */ +/** Varint encoding and decoding, in QUIC's format and moq-transport's leading-ones format. */ export * as Varint from "./varint.ts"; diff --git a/js/net/src/stream.test.ts b/js/net/src/stream.test.ts index 8ecdede0ca..1c54ff92c8 100644 --- a/js/net/src/stream.test.ts +++ b/js/net/src/stream.test.ts @@ -13,6 +13,7 @@ import { import { Version } from "./ietf/version.ts"; import { type Cursor, Reader, Stream, Writer } from "./stream.ts"; import { TimeoutError } from "./util/timeout.ts"; +import { U64 } from "./util/u64.ts"; // Helper to create a writable stream that captures written data function createTestWritableStream(): { stream: WritableStream; written: Uint8Array[] } { @@ -796,3 +797,55 @@ for (const [version, tooFarBehind] of [ if (version !== undefined) expect((err as StreamError).message).toContain("70"); }); } + +test("Writer and Reader varint round-trip every size in both formats", async () => { + const values = [0n, 63n, 64n, 127n, 128n, 2n ** 14n, 2n ** 21n, 2n ** 28n, 2n ** 30n, 2n ** 32n, 2n ** 35n]; + values.push(2n ** 42n, 2n ** 49n, 2n ** 53n, 2n ** 56n, 2n ** 62n - 1n); + for (const version of [undefined, Version.DRAFT_16, Version.DRAFT_17, Version.DRAFT_19]) { + // Only leading-ones varints reach past 62 bits. + const all = + version === Version.DRAFT_17 || version === Version.DRAFT_19 + ? [...values, 2n ** 62n, 2n ** 64n - 1n] + : values; + const { stream, written } = createTestWritableStream(); + const writer = new Writer(stream, version); + for (const value of all) await writer.varint(U64.fromBigInt(value)); + writer.close(); + await writer.closed; + + const reader = new Reader(undefined, concatChunks(written), version); + for (const value of all) expect((await reader.varint()).toBigInt()).toBe(value); + expect(await reader.done()).toBe(true); + } +}); + +test("Writer varint refuses a QUIC varint past 62 bits before emitting bytes", async () => { + const { stream, written } = createTestWritableStream(); + const writer = new Writer(stream); + await expect(writer.varint(U64.fromBigInt(2n ** 62n))).rejects.toThrow(/larger than 62-bits/); + await expect(writer.varint(U64.MAX)).rejects.toThrow(/larger than 62-bits/); + expect(written).toEqual([]); +}); + +test("Reader u53 decodes every size in both formats, one byte per chunk", async () => { + const values = [0, 63, 64, 127, 128, 16383, 16384, 2 ** 21, 2 ** 28 - 1, 2 ** 28, 2 ** 30 - 1, 2 ** 30]; + values.push(2 ** 35, 2 ** 42, 2 ** 49, Number.MAX_SAFE_INTEGER); + for (const version of [undefined, Version.DRAFT_17]) { + const { stream, written } = createTestWritableStream(); + const writer = new Writer(stream, version); + for (const value of values) await writer.u53(value); + writer.close(); + await writer.closed; + + const bytes = concatChunks(written); + const chunked = new ReadableStream({ + start(controller) { + for (const byte of bytes) controller.enqueue(new Uint8Array([byte])); + controller.close(); + }, + }); + const reader = new Reader(chunked, undefined, version); + for (const value of values) expect(await reader.u53()).toBe(value); + expect(await reader.done()).toBe(true); + } +}); diff --git a/js/net/src/stream.ts b/js/net/src/stream.ts index 8093e4495c..47e3d374d5 100644 --- a/js/net/src/stream.ts +++ b/js/net/src/stream.ts @@ -3,8 +3,20 @@ import { fromTransport, StreamCode, StreamError, toStreamCode, toTransport } fro import type { IetfVersion } from "./ietf/version.ts"; import { Version } from "./ietf/version.ts"; import { TimeoutError, withTimeout } from "./util/timeout.ts"; +import { POW32, toBigInt, toNumber, U64 } from "./util/u64.ts"; import { decodeUtf8 } from "./util/utf8.ts"; -import * as Varint from "./varint.ts"; +import { + lengthLeadingOnes, + lengthQuic, + parts, + peekLeadingOnes, + peekQuic, + readLeadingOnes, + readQuic, + split, + writeLeadingOnes, + writeQuic, +} from "./util/varint.ts"; // Decode raw transport errors before mapping so they cannot bypass the negotiated // registry. Ordinary errors already send 0 and retain their local identity. @@ -367,6 +379,10 @@ export class Reader { return this.decode(U62); } + async varint(): Promise { + return this.decode(VARINT); + } + // Returns false if there is more data to read, blocking if it hasn't been received yet. async done(): Promise { if (this.#buffer.byteLength > 0 || this.#chunked > 0) return false; @@ -411,10 +427,20 @@ export class Cursor { readonly version?: IetfVersion; #buffer: Uint8Array; #offset = 0; + // Resolved once, since every varint read branches on it. + #leadingOnes: boolean; + // First bytes below this are a whole 1-byte varint, and below this + 0x40 a 2-byte one whose + // value is the low 6 bits and the next byte. Both formats share that shape; only the bound moves. + #short: number; + // First bytes below this are a varint of at most 4 bytes: 0xc0 for QUIC, 0xf0 for leading-ones. + #word: number; constructor(buffer: Uint8Array, version?: IetfVersion) { this.#buffer = buffer; this.version = version; + this.#leadingOnes = isLeadingOnes(version); + this.#short = this.#leadingOnes ? 0x80 : 0x40; + this.#word = this.#leadingOnes ? 0xf0 : 0xc0; } /** How many bytes have been read. */ @@ -484,69 +510,66 @@ export class Cursor { return (b[o] << 8) | b[o + 1]; } - // Returns a Number using 53-bits, the max Javascript can use for integer math. + /** Read a varint as a `number`, throwing if it is above `Number.MAX_SAFE_INTEGER`. */ u53(): number { - // Most varints fit in 4 bytes, which decode without a bigint. - if (!isLeadingOnes(this.version)) { - this.#ensure(1); - const b = this.#buffer; - const o = this.#offset; - const size = 1 << (b[o] >> 6); - if (size < 8) { - this.#ensure(size); - this.#offset += size; - if (size === 1) return b[o] & 0x3f; - if (size === 2) return ((b[o] & 0x3f) << 8) | b[o + 1]; - return (b[o] & 0x3f) * 2 ** 24 + ((b[o + 1] << 16) | (b[o + 2] << 8) | b[o + 3]); - } + // Most varints are 1 or 2 bytes, which skip the general decode. + this.#ensure(1); + const b = this.#buffer; + const o = this.#offset; + const first = b[o]; + if (first < this.#short) { + this.#offset = o + 1; + return first; } - - const v = this.u62(); - if (v > Varint.MAX_U53) { - throw new Error(`value larger than 53-bits: ${v.toString()}`); + if (first < this.#short + 0x40) { + this.#ensure(2); + this.#offset = o + 2; + return ((first & 0x3f) << 8) | b[o + 1]; } - return Number(v); + // Up to 4 bytes still fits 28 (leading-ones) or 30 (QUIC) bits, with no upper half. + if (first < this.#word) { + const size = this.#leadingOnes ? peekLeadingOnes(first) : 4; + this.#ensure(size); + this.#offset = o + size; + if (size === 3) return ((first & 0x1f) << 16) | (b[o + 1] << 8) | b[o + 2]; + return ((first & (this.#leadingOnes ? 0x0f : 0x3f)) << 24) | (b[o + 1] << 16) | (b[o + 2] << 8) | b[o + 3]; + } + const lo = this.#varint(); + return toNumber(parts.hi, lo); } - // NOTE: Returns a bigint instead of a number since it may be larger than 53-bits + /** Read a varint as a bigint. A leading-ones varint may exceed 62 bits. */ u62(): bigint { - return isLeadingOnes(this.version) ? this.#leadingOnes() : this.#quicVarint(); + const lo = this.#varint(); + return toBigInt(parts.hi, lo); } - #quicVarint(): bigint { - this.#ensure(1); - const size = 1 << (this.#buffer[this.#offset] >> 6); - if (size < 8) return BigInt(this.u53()); - - const slice = this.read(8); - const view = new DataView(slice.buffer, slice.byteOffset, slice.byteLength); - return view.getBigUint64(0) & 0x3fffffffffffffffn; + /** Read a varint. */ + varint(): U64 { + const lo = this.#varint(); + return new U64(parts.hi, lo); } - #leadingOnes(): bigint { + // Decode the next varint in the version's format, returning its lower half and leaving the upper in `parts`. + #varint(): number { this.#ensure(1); - const b = this.#buffer[this.#offset]; - - // Count leading 1-bits - let ones = 0; - for (let bit = 7; bit >= 0; bit--) { - if (b & (1 << bit)) ones++; - else break; - } - - // 1111110x is a 7-byte form. Draft-17 rejects it; draft-18+ allows it per #1595. - if (ones === 6 && this.version === Version.DRAFT_17) { - throw new Error("invalid leading-ones varint: 1111110x prefix is reserved on draft-17"); + const b = this.#buffer; + const o = this.#offset; + let size: number; + if (this.#leadingOnes) { + size = peekLeadingOnes(b[o]); + // 1111110x is a 7-byte form. Draft-17 rejects it; draft-18+ allows it per #1595. + if (size === 7 && this.version === Version.DRAFT_17) { + throw new Error("invalid leading-ones varint: 1111110x prefix is reserved on draft-17"); + } + this.#ensure(size); + this.#offset += size; + return readLeadingOnes(b, o, size); } - - let totalSize: number; - if (ones <= 5) totalSize = ones + 1; - else if (ones === 6) totalSize = 7; - else if (ones === 7) totalSize = 8; - else totalSize = 9; // ones === 8 - - const [value] = Varint.decodeLeadingOnes(this.read(totalSize)); - return value; + size = peekQuic(b[o]); + this.#ensure(size); + this.#offset += size; + return readQuic(b, o, size); } } @@ -557,6 +580,7 @@ const U8 = (c: Cursor) => c.u8(); const U16 = (c: Cursor) => c.u16(); const U53 = (c: Cursor) => c.u53(); const U62 = (c: Cursor) => c.u62(); +const VARINT = (c: Cursor) => c.varint(); // Writer wraps a stream and writes chunks of data export class Writer { @@ -564,8 +588,7 @@ export class Writer { #stream: WritableStream; #closed?: Promise; - // Scratch buffer for writing varints. - // Fixed at 9 bytes (leading-ones max). + // Scratch buffer for each primitive write, sized for the longest (a 9-byte leading-ones varint). #scratch: ArrayBuffer; version?: IetfVersion; @@ -611,7 +634,7 @@ export class Writer { throw new Error(`overflow, value larger than 32-bits: ${v.toString()}`); } - // We don't use a VarInt, so it always takes 4 bytes. + // We don't use a varint, so it always takes 4 bytes. // This could be improved but nothing is standardized yet. await this.write(setInt32(this.#scratch, v)); } @@ -620,19 +643,28 @@ export class Writer { if (!Number.isSafeInteger(v) || v < 0) { throw new RangeError(`invalid u53: ${v}`); } - if (isLeadingOnes(this.version)) { - await this.write(Varint.encodeLeadingOnesTo(this.#scratch, v)); - } else { - await this.write(Varint.encodeTo(this.#scratch, v)); - } + await this.#varint(Math.floor(v / POW32), v >>> 0); } async u62(v: bigint) { + const lo = split(v); + await this.#varint(parts.hi, lo); + } + + async varint(v: U64) { + await this.#varint(v.hi, v.lo); + } + + #varint(hi: number, lo: number): Promise { + let buf: Uint8Array; if (isLeadingOnes(this.version)) { - await this.write(Varint.encodeLeadingOnesTo(this.#scratch, v)); + buf = new Uint8Array(this.#scratch, 0, lengthLeadingOnes(hi, lo)); + writeLeadingOnes(buf, hi, lo, buf.length); } else { - await this.write(Varint.encodeTo(this.#scratch, v)); + buf = new Uint8Array(this.#scratch, 0, lengthQuic(hi, lo)); + writeQuic(buf, hi, lo, buf.length); } + return this.write(buf); } async write(v: Uint8Array) { diff --git a/js/net/src/util/u64.test.ts b/js/net/src/util/u64.test.ts new file mode 100644 index 0000000000..8b89e361bb --- /dev/null +++ b/js/net/src/util/u64.test.ts @@ -0,0 +1,50 @@ +import { expect, test } from "bun:test"; +import { U64 } from "./u64.ts"; + +test("U64 converts numbers at the 32-bit and 53-bit boundaries", () => { + for (const n of [0, 2 ** 30, 2 ** 32 - 1, 2 ** 32, Number.MAX_SAFE_INTEGER]) { + const v = U64.fromNumber(n); + expect(v.toNumber()).toBe(n); + expect(v.toBigInt()).toBe(BigInt(n)); + expect(v.toString()).toBe(String(n)); + } + expect(U64.fromNumber(2 ** 32 + 5)).toEqual(new U64(1, 5)); +}); + +test("U64.fromNumber rejects anything but a non-negative safe integer", () => { + for (const n of [-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, 2 ** 53]) { + expect(() => U64.fromNumber(n)).toThrow(RangeError); + } +}); + +test("U64 holds 64 bits but converts to number only up to 2^53 - 1", () => { + const max = U64.fromBigInt(2n ** 64n - 1n); + expect(max).toEqual(U64.MAX); + expect(max.toBigInt()).toBe(2n ** 64n - 1n); + expect(max.toString()).toBe("18446744073709551615"); + expect(() => max.toNumber()).toThrow(/larger than 53-bits: 18446744073709551615/); + expect(() => U64.fromBigInt(2n ** 53n).toNumber()).toThrow(RangeError); + + expect(() => U64.fromBigInt(2n ** 64n)).toThrow(RangeError); + expect(() => U64.fromBigInt(-1n)).toThrow(RangeError); + expect(() => new U64(2 ** 32, 0)).toThrow(RangeError); + expect(() => new U64(0, 2 ** 32)).toThrow(RangeError); + expect(() => new U64(0, -1)).toThrow(RangeError); +}); + +test("U64 compares and adds without converting", () => { + const a = U64.fromNumber(2 ** 32 - 1); + const b = a.add(1); + expect(b).toEqual(new U64(1, 0)); + expect(a.compare(b)).toBeLessThan(0); + expect(b.compare(a)).toBeGreaterThan(0); + expect(a.compare(U64.fromNumber(2 ** 32 - 1))).toBe(0); + expect(U64.MAX.compare(U64.ZERO)).toBeGreaterThan(0); + expect(a.equals(b)).toBe(false); + expect(b.equals(new U64(1, 0))).toBe(true); + + expect(U64.ZERO.add(Number.MAX_SAFE_INTEGER).toNumber()).toBe(Number.MAX_SAFE_INTEGER); + expect(U64.fromBigInt(2n ** 64n - 2n).add(1)).toEqual(U64.MAX); + expect(() => U64.MAX.add(1)).toThrow(RangeError); + expect(() => a.add(-1)).toThrow(RangeError); +}); diff --git a/js/net/src/util/u64.ts b/js/net/src/util/u64.ts new file mode 100644 index 0000000000..cc3484023b --- /dev/null +++ b/js/net/src/util/u64.ts @@ -0,0 +1,88 @@ +// An unsigned 64-bit integer as two u32 halves, so nothing on the hot path needs a BigInt. +// Package-internal. Varints are only its wire encoding; see ./varint.ts. + +/** 2^32, the weight of the upper half. */ +export const POW32 = 0x1_0000_0000; +// An upper half at or above this is past `Number.MAX_SAFE_INTEGER`. +const SAFE_HI = 2 ** 21; + +/** Join two halves into a `number`, throwing if it is above `Number.MAX_SAFE_INTEGER` rather than rounding. */ +export function toNumber(hi: number, lo: number): number { + if (hi >= SAFE_HI) throw new RangeError(`value larger than 53-bits: ${toBigInt(hi, lo)}`); + return hi * POW32 + lo; +} + +/** Join two halves into a bigint, exact up to 64 bits. */ +export function toBigInt(hi: number, lo: number): bigint { + return hi === 0 ? BigInt(lo) : (BigInt(hi) << 32n) | BigInt(lo); +} + +/** + * An unsigned 64-bit integer, held as two 32-bit halves so it encodes and decodes without a BigInt. + * Converting to a `number` throws past 2^53 rather than rounding. + */ +export class U64 { + /** The value 0. */ + static readonly ZERO = new U64(0, 0); + /** The largest value, 2^64 - 1. */ + static readonly MAX = new U64(POW32 - 1, POW32 - 1); + + /** The upper 32 bits. */ + readonly hi: number; + /** The lower 32 bits. */ + readonly lo: number; + + /** Join the upper and lower 32 bits, throwing if either is not a u32. */ + constructor(hi: number, lo: number) { + if (!Number.isInteger(hi) || hi < 0 || hi >= POW32) throw new RangeError(`invalid upper half: ${hi}`); + if (!Number.isInteger(lo) || lo < 0 || lo >= POW32) throw new RangeError(`invalid lower half: ${lo}`); + this.hi = hi; + this.lo = lo; + } + + /** Convert a non-negative safe integer, throwing on anything else. */ + static fromNumber(v: number): U64 { + if (!Number.isSafeInteger(v) || v < 0) throw new RangeError(`invalid u64: ${v}`); + return new U64(Math.floor(v / POW32), v >>> 0); + } + + /** Convert a bigint, throwing unless it is in [0, 2^64). */ + static fromBigInt(v: bigint): U64 { + if (v < 0n || v >> 64n) throw new RangeError(`invalid u64: ${v}`); + return new U64(Number(v >> 32n), Number(v & 0xffffffffn)); + } + + /** Convert to a `number`, throwing if it is above `Number.MAX_SAFE_INTEGER`. */ + toNumber(): number { + return toNumber(this.hi, this.lo); + } + + /** Convert to a bigint, exactly. */ + toBigInt(): bigint { + return toBigInt(this.hi, this.lo); + } + + /** The value in decimal. */ + toString(): string { + return this.hi < SAFE_HI ? String(this.hi * POW32 + this.lo) : this.toBigInt().toString(); + } + + /** Negative if this is less than `other`, zero if equal, positive if greater. */ + compare(other: U64): number { + return this.hi - other.hi || this.lo - other.lo; + } + + /** Whether this equals `other`. */ + equals(other: U64): boolean { + return this.hi === other.hi && this.lo === other.lo; + } + + /** This plus a non-negative safe integer, throwing if the sum reaches 2^64. */ + add(delta: number): U64 { + if (!Number.isSafeInteger(delta) || delta < 0) throw new RangeError(`invalid delta: ${delta}`); + const lo = this.lo + (delta >>> 0); + const hi = this.hi + Math.floor(delta / POW32) + (lo >= POW32 ? 1 : 0); + if (hi >= POW32) throw new RangeError(`overflow, ${this} + ${delta} exceeds 64 bits`); + return new U64(hi, lo >>> 0); + } +} diff --git a/js/net/src/util/varint.test.ts b/js/net/src/util/varint.test.ts new file mode 100644 index 0000000000..0eeaa0bcb4 --- /dev/null +++ b/js/net/src/util/varint.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from "bun:test"; +import * as Varint from "../varint.ts"; +import { U64 } from "./u64.ts"; +import { + lengthLeadingOnes, + lengthQuic, + parts, + peekLeadingOnes, + peekQuic, + readLeadingOnes, + readQuic, + writeLeadingOnes, + writeQuic, +} from "./varint.ts"; + +// Each boundary, and the expected size in QUIC form (undefined where QUIC can't hold it) and in leading-ones form. +const boundaries: [bigint, number | undefined, number][] = [ + [2n ** 30n - 1n, 4, 5], + [2n ** 30n, 8, 5], + [2n ** 53n - 1n, 8, 8], + [2n ** 53n, 8, 8], + [2n ** 62n - 1n, 8, 9], + [2n ** 62n, undefined, 9], + [2n ** 64n - 1n, undefined, 9], +]; + +test("Both formats round-trip the 2^30, 2^53, 2^62, and 2^64 boundaries", () => { + const buf = new Uint8Array(9); + for (const [value, quic, leadingOnes] of boundaries) { + const v = U64.fromBigInt(value); + + if (quic === undefined) { + expect(() => lengthQuic(v.hi, v.lo)).toThrow(/larger than 62-bits/); + expect(() => Varint.encode(value)).toThrow(/larger than 62-bits/); + } else { + expect(lengthQuic(v.hi, v.lo)).toBe(quic); + writeQuic(buf, v.hi, v.lo, quic); + expect(peekQuic(buf[0])).toBe(quic); + const lo = readQuic(buf, 0, quic); + expect(new U64(parts.hi, lo)).toEqual(v); + expect(Varint.decodeBigInt(Varint.encode(value))[0]).toBe(value); + } + + expect(lengthLeadingOnes(v.hi, v.lo)).toBe(leadingOnes); + writeLeadingOnes(buf, v.hi, v.lo, leadingOnes); + expect(peekLeadingOnes(buf[0])).toBe(leadingOnes); + const lo = readLeadingOnes(buf, 0, leadingOnes); + expect(new U64(parts.hi, lo)).toEqual(v); + expect(Varint.decodeLeadingOnes(Varint.encodeLeadingOnes(value))[0]).toBe(value); + } +}); diff --git a/js/net/src/util/varint.ts b/js/net/src/util/varint.ts new file mode 100644 index 0000000000..dd355ad515 --- /dev/null +++ b/js/net/src/util/varint.ts @@ -0,0 +1,188 @@ +// The varint wire codec on 32-bit halves (see ./u64.ts), so neither format needs a BigInt. +// Package-internal: the public API in ../varint.ts and the stream Cursor and Writer are built on it. +// +// QUIC (RFC 9000 Section 16): the top two bits give the size. +// 00xxxxxx → 1 byte, 01 → 2, 10 → 4, 11 → 8 (62 bits) +// +// Leading-ones (moq-transport draft-17+ Section 1.4.1): the leading 1-bits give the size. +// 0xxxxxxx → 1 byte (7 bits) +// 10xxxxxx + 1B → 2 bytes (14 bits) +// 110xxxxx + 2B → 3 bytes (21 bits) +// 1110xxxx + 3B → 4 bytes (28 bits) +// 11110xxx + 4B → 5 bytes (35 bits) +// 111110xx + 5B → 6 bytes (42 bits) +// 1111110x + 6B → 7 bytes (49 bits), draft-18+ only (invalid in draft-17 per #1595) +// 11111110 + 7B → 8 bytes (56 bits) +// 11111111 + 8B → 9 bytes (64 bits) + +import { POW32, toBigInt } from "./u64.ts"; + +/** + * The upper half of the last value decoded or split. Each returns its lower half and leaves the + * upper one here rather than allocating a pair; read it before the next call. + */ +export const parts = { hi: 0 }; + +/** Split a non-negative integer of up to 64 bits, returning the lower half. The encoder bounds it. */ +export function split(v: number | bigint): number { + if (typeof v === "number") { + if (!Number.isInteger(v)) throw new RangeError(`not an integer: ${v}`); + if (v < 0) throw new RangeError(`underflow, value is negative: ${v}`); + if (v >= POW32 * POW32) throw new RangeError(`value exceeds 64 bits: ${v}`); + parts.hi = Math.floor(v / POW32); + return v >>> 0; + } + if (v < 0n) throw new RangeError(`underflow, value is negative: ${v}`); + if (v >> 64n) throw new RangeError(`value exceeds 64 bits: ${v}`); + parts.hi = Number(v >> 32n); + return Number(v & 0xffffffffn); +} + +function u32(buf: Uint8Array, o: number): number { + return ((buf[o] << 24) | (buf[o + 1] << 16) | (buf[o + 2] << 8) | buf[o + 3]) >>> 0; +} + +function setU32(buf: Uint8Array, o: number, v: number) { + buf[o] = v >>> 24; + buf[o + 1] = v >>> 16; + buf[o + 2] = v >>> 8; + buf[o + 3] = v; +} + +/** The size of a QUIC varint, from its first byte. */ +export function peekQuic(first: number): number { + return 1 << (first >> 6); +} + +/** Decode a QUIC varint of `size` bytes at `o`, returning the lower half and setting `parts.hi`. The bytes must be there. */ +export function readQuic(buf: Uint8Array, o: number, size: number): number { + const b = buf[o] & 0x3f; + if (size === 8) { + parts.hi = (b << 24) | (buf[o + 1] << 16) | (buf[o + 2] << 8) | buf[o + 3]; + return u32(buf, o + 4); + } + parts.hi = 0; + if (size === 1) return b; + if (size === 2) return (b << 8) | buf[o + 1]; + return (b << 24) | (buf[o + 1] << 16) | (buf[o + 2] << 8) | buf[o + 3]; +} + +/** The size of `hi`/`lo` as a QUIC varint, throwing at 2^62 or above. */ +export function lengthQuic(hi: number, lo: number): number { + if (hi === 0) { + if (lo < 0x40) return 1; + if (lo < 0x4000) return 2; + if (lo < 0x4000_0000) return 4; + } + if (hi >= 0x4000_0000) throw new RangeError(`overflow, value larger than 62-bits: ${toBigInt(hi, lo)}`); + return 8; +} + +/** Encode a QUIC varint of {@link lengthQuic} `size` at the start of `dst`. */ +export function writeQuic(dst: Uint8Array, hi: number, lo: number, size: number) { + if (size === 1) { + dst[0] = lo; + } else if (size === 2) { + dst[0] = 0x40 | (lo >>> 8); + dst[1] = lo; + } else if (size === 4) { + setU32(dst, 0, 0x8000_0000 | lo); + } else { + setU32(dst, 0, 0xc000_0000 | hi); + setU32(dst, 4, lo); + } +} + +/** The size of a leading-ones varint, from its first byte. */ +export function peekLeadingOnes(first: number): number { + return Math.clz32(~(first << 24)) + 1; +} + +/** Decode a leading-ones varint of `size` bytes at `o`, returning the lower half and setting `parts.hi`. The bytes must be there. */ +export function readLeadingOnes(buf: Uint8Array, o: number, size: number): number { + const b = buf[o]; + switch (size) { + case 1: + parts.hi = 0; + return b; + case 2: + parts.hi = 0; + return ((b & 0x3f) << 8) | buf[o + 1]; + case 3: + parts.hi = 0; + return ((b & 0x1f) << 16) | (buf[o + 1] << 8) | buf[o + 2]; + case 4: + parts.hi = 0; + return ((b & 0x0f) << 24) | (buf[o + 1] << 16) | (buf[o + 2] << 8) | buf[o + 3]; + case 5: + parts.hi = b & 0x07; + return u32(buf, o + 1); + case 6: + parts.hi = ((b & 0x03) << 8) | buf[o + 1]; + return u32(buf, o + 2); + case 7: + parts.hi = ((b & 0x01) << 16) | (buf[o + 1] << 8) | buf[o + 2]; + return u32(buf, o + 3); + case 8: + parts.hi = (buf[o + 1] << 16) | (buf[o + 2] << 8) | buf[o + 3]; + return u32(buf, o + 4); + default: + parts.hi = u32(buf, o + 1); + return u32(buf, o + 5); + } +} + +/** + * The size of `hi`/`lo` as a leading-ones varint, in its shortest form. The 7-byte form is skipped: + * draft-17 rejects it, and the 8-byte form is valid everywhere. + */ +export function lengthLeadingOnes(hi: number, lo: number): number { + if (hi === 0) { + if (lo < 0x80) return 1; + if (lo < 0x4000) return 2; + if (lo < 0x20_0000) return 3; + if (lo < 0x1000_0000) return 4; + } + if (hi < 0x8) return 5; + if (hi < 0x400) return 6; + if (hi < 0x100_0000) return 8; + return 9; +} + +/** Encode a leading-ones varint of {@link lengthLeadingOnes} `size` at the start of `dst`. */ +export function writeLeadingOnes(dst: Uint8Array, hi: number, lo: number, size: number) { + switch (size) { + case 1: + dst[0] = lo; + return; + case 2: + dst[0] = 0x80 | (lo >>> 8); + dst[1] = lo; + return; + case 3: + dst[0] = 0xc0 | (lo >>> 16); + dst[1] = lo >>> 8; + dst[2] = lo; + return; + case 4: + setU32(dst, 0, 0xe000_0000 | lo); + return; + case 5: + dst[0] = 0xf0 | hi; + setU32(dst, 1, lo); + return; + case 6: + dst[0] = 0xf8 | (hi >>> 8); + dst[1] = hi; + setU32(dst, 2, lo); + return; + case 8: + setU32(dst, 0, 0xfe00_0000 | hi); + setU32(dst, 4, lo); + return; + default: + dst[0] = 0xff; + setU32(dst, 1, hi); + setU32(dst, 5, lo); + } +} diff --git a/js/net/src/varint.test.ts b/js/net/src/varint.test.ts index a0b6894442..ff28f83220 100644 --- a/js/net/src/varint.test.ts +++ b/js/net/src/varint.test.ts @@ -135,3 +135,11 @@ test("Varint boundary values", () => { expect(decoded).toBe(value); } }); + +test("Varint decode throws past 2^53 - 1 instead of rounding", () => { + expect(Varint.decode(Varint.encode(2n ** 53n - 1n))[0]).toBe(Number.MAX_SAFE_INTEGER); + for (const value of [2n ** 53n, 2n ** 53n + 1n, 2n ** 62n - 1n]) { + expect(() => Varint.decode(Varint.encode(value))).toThrow(`value larger than 53-bits: ${value}`); + expect(Varint.decodeBigInt(Varint.encode(value))[0]).toBe(value); + } +}); diff --git a/js/net/src/varint.ts b/js/net/src/varint.ts index afeb66ec72..1d59247558 100644 --- a/js/net/src/varint.ts +++ b/js/net/src/varint.ts @@ -1,10 +1,24 @@ /** - * QUIC variable-length integer encoding and decoding. + * Variable-length integers: QUIC's (RFC 9000 Section 16) and moq-transport's leading-ones form. * https://www.rfc-editor.org/rfc/rfc9000#section-16 * * @module */ +import { toBigInt, toNumber } from "./util/u64.ts"; +import { + lengthLeadingOnes, + lengthQuic, + parts, + peekLeadingOnes, + peekQuic, + readLeadingOnes, + readQuic, + split, + writeLeadingOnes, + writeQuic, +} from "./util/varint.ts"; + /** Largest value that fits in a 1-byte varint (6 bits). */ export const MAX_U6 = 2 ** 6 - 1; /** Largest value that fits in a 2-byte varint (14 bits). */ @@ -14,86 +28,26 @@ export const MAX_U30 = 2 ** 30 - 1; /** Largest value representable without precision loss (`Number.MAX_SAFE_INTEGER`, 53 bits). */ export const MAX_U53 = Number.MAX_SAFE_INTEGER; -// Leading-ones varint encoding/decoding (draft-17 Section 1.4.1) -// Encoding scheme: -// 0xxxxxxx → 1 byte (7 bits) -// 10xxxxxx + 1B → 2 bytes (14 bits) -// 110xxxxx + 2B → 3 bytes (21 bits) -// 1110xxxx + 3B → 4 bytes (28 bits) -// 11110xxx + 4B → 5 bytes (35 bits) -// 111110xx + 5B → 6 bytes (42 bits) -// 1111110x + 6B → 7 bytes (49 bits), draft-18+ only (invalid in draft-17 per #1595) -// 11111110 + 7B → 8 bytes (56 bits) -// 11111111 + 8B → 9 bytes (64 bits) - -const MAX_U64 = (1n << 64n) - 1n; +// The size of the varint at the start of `buf`, throwing unless all of it is there. +function sizeOf(buf: Uint8Array, size: (first: number) => number): number { + if (buf.length === 0) throw new Error("buffer is empty"); + const n = size(buf[0]); + if (buf.length < n) throw new Error(`buffer too short: need ${n} bytes, have ${buf.length}`); + return n; +} /** Number of bytes needed to encode a value in the leading-ones varint format. */ export function sizeLeadingOnes(v: number | bigint): number { - const b = BigInt(v); - if (b < 0n) throw new RangeError(`value is negative: ${v}`); - if (b > MAX_U64) throw new RangeError(`value exceeds 64 bits: ${v}`); - if (b < 1n << 7n) return 1; - if (b < 1n << 14n) return 2; - if (b < 1n << 21n) return 3; - if (b < 1n << 28n) return 4; - if (b < 1n << 35n) return 5; - if (b < 1n << 42n) return 6; - if (b < 1n << 56n) return 8; - return 9; + const lo = split(v); + return lengthLeadingOnes(parts.hi, lo); } /** Encode a value in leading-ones varint format into the provided buffer, returning the written subarray. */ export function encodeLeadingOnesTo(dst: ArrayBuffer, v: number | bigint): Uint8Array { - const x = BigInt(v); - if (x < 0n) throw new RangeError(`underflow, value is negative: ${v}`); - if (x > MAX_U64) throw new RangeError(`value exceeds 64 bits: ${v}`); - - const view = new DataView(dst); - - if (x < 1n << 7n) { - view.setUint8(0, Number(x)); - return new Uint8Array(dst, 0, 1); - } - if (x < 1n << 14n) { - view.setUint8(0, 0x80 | Number(x >> 8n)); - view.setUint8(1, Number(x & 0xffn)); - return new Uint8Array(dst, 0, 2); - } - if (x < 1n << 21n) { - view.setUint8(0, 0xc0 | Number(x >> 16n)); - view.setUint16(1, Number(x & 0xffffn)); - return new Uint8Array(dst, 0, 3); - } - if (x < 1n << 28n) { - view.setUint8(0, 0xe0 | Number(x >> 24n)); - view.setUint8(1, Number((x >> 16n) & 0xffn)); - view.setUint16(2, Number(x & 0xffffn)); - return new Uint8Array(dst, 0, 4); - } - if (x < 1n << 35n) { - view.setUint8(0, 0xf0 | Number(x >> 32n)); - view.setUint32(1, Number(x & 0xffffffffn)); - return new Uint8Array(dst, 0, 5); - } - if (x < 1n << 42n) { - view.setUint8(0, 0xf8 | Number(x >> 40n)); - view.setUint8(1, Number((x >> 32n) & 0xffn)); - view.setUint32(2, Number(x & 0xffffffffn)); - return new Uint8Array(dst, 0, 6); - } - if (x < 1n << 56n) { - // 11111110 + 7 bytes - view.setUint8(0, 0xfe); - view.setUint8(1, Number((x >> 48n) & 0xffn)); - view.setUint16(2, Number((x >> 32n) & 0xffffn)); - view.setUint32(4, Number(x & 0xffffffffn)); - return new Uint8Array(dst, 0, 8); - } - // 11111111 + 8 bytes - view.setUint8(0, 0xff); - view.setBigUint64(1, x); - return new Uint8Array(dst, 0, 9); + const lo = split(v); + const buf = new Uint8Array(dst, 0, lengthLeadingOnes(parts.hi, lo)); + writeLeadingOnes(buf, parts.hi, lo, buf.length); + return buf; } /** Encode a value in leading-ones varint format into a freshly allocated buffer. */ @@ -101,90 +55,15 @@ export function encodeLeadingOnes(v: number | bigint): Uint8Array { return encodeLeadingOnesTo(new ArrayBuffer(9), v); } -/** Decode a leading-ones varint, returning the value and the remaining buffer. */ +/** + * Decode a leading-ones varint, returning the value and the remaining buffer. + * + * Accepts the 7-byte form, which only draft-18+ allows, since there is no version here to check. + */ export function decodeLeadingOnes(buf: Uint8Array): [bigint, Uint8Array] { - if (buf.length === 0) throw new Error("buffer is empty"); - - const b = buf[0]; - // Count leading 1-bits - let ones = 0; - for (let bit = 7; bit >= 0; bit--) { - if (b & (1 << bit)) ones++; - else break; - } - - // 1111110x is a 7-byte form: invalid on draft-17, allowed on draft-18+ per #1595. - // This standalone decoder is permissive (Postel-style) since we lack version context here. - - let totalSize: number; - if (ones <= 5) totalSize = ones + 1; - else if (ones === 6) totalSize = 7; - else if (ones === 7) totalSize = 8; - else totalSize = 9; // ones === 8 - - if (buf.length < totalSize) { - throw new Error(`buffer too short: need ${totalSize} bytes, have ${buf.length}`); - } - - const view = new DataView(buf.buffer, buf.byteOffset, totalSize); - const remain = buf.subarray(totalSize); - let value: bigint; - - switch (ones) { - case 0: - value = BigInt(b); - break; - case 1: - value = (BigInt(b & 0x3f) << 8n) | BigInt(buf[1]); - break; - case 2: - value = (BigInt(b & 0x1f) << 16n) | BigInt(view.getUint16(1)); - break; - case 3: - value = (BigInt(b & 0x0f) << 24n) | (BigInt(buf[1]) << 16n) | (BigInt(buf[2]) << 8n) | BigInt(buf[3]); - break; - case 4: - value = (BigInt(b & 0x07) << 32n) | BigInt(view.getUint32(1)); - break; - case 5: - value = - (BigInt(b & 0x03) << 40n) | - (BigInt(buf[1]) << 32n) | - (BigInt(buf[2]) << 24n) | - (BigInt(buf[3]) << 16n) | - (BigInt(buf[4]) << 8n) | - BigInt(buf[5]); - break; - case 6: { - // 1111110x + 6 bytes = 49 bits (draft-18+) - value = - (BigInt(b & 0x01) << 48n) | - (BigInt(buf[1]) << 40n) | - (BigInt(buf[2]) << 32n) | - (BigInt(buf[3]) << 24n) | - (BigInt(buf[4]) << 16n) | - (BigInt(buf[5]) << 8n) | - BigInt(buf[6]); - break; - } - case 7: { - // 11111110 + 7 bytes = 56 usable bits - const hi = new Uint8Array(8); - hi[0] = 0; - hi.set(buf.subarray(1, 8), 1); - value = new DataView(hi.buffer).getBigUint64(0); - break; - } - case 8: { - // 11111111 + 8 bytes = 64 bits - value = new DataView(buf.buffer, buf.byteOffset + 1, 8).getBigUint64(0); - break; - } - default: - throw new Error("impossible"); - } - - return [value, remain]; + const size = sizeOf(buf, peekLeadingOnes); + const lo = readLeadingOnes(buf, 0, size); + return [toBigInt(parts.hi, lo), buf.subarray(size)]; } /** @@ -198,60 +77,19 @@ export function size(v: number): number { throw new Error(`overflow, value larger than 53-bits: ${v}`); } -// Helper functions for writing to an ArrayBuffer -function setUint8(dst: ArrayBuffer, v: number): Uint8Array { - const buffer = new Uint8Array(dst, 0, 1); - buffer[0] = v; - return buffer; -} - -function setUint16(dst: ArrayBuffer, v: number): Uint8Array { - const view = new DataView(dst, 0, 2); - view.setUint16(0, v); - return new Uint8Array(view.buffer, view.byteOffset, view.byteLength); -} - -function setUint32(dst: ArrayBuffer, v: number): Uint8Array { - const view = new DataView(dst, 0, 4); - view.setUint32(0, v); - return new Uint8Array(view.buffer, view.byteOffset, view.byteLength); -} - -function setUint64(dst: ArrayBuffer, v: bigint): Uint8Array { - const view = new DataView(dst, 0, 8); - view.setBigUint64(0, v); - return new Uint8Array(view.buffer, view.byteOffset, view.byteLength); -} - -const MAX_U62 = 2n ** 62n - 1n; - /** - * Encodes a number or bigint into a scratch buffer. - * Used by stream.ts to avoid allocations. + * Encodes a value as a QUIC variable-length integer into the provided buffer, + * returning the written subarray. */ export function encodeTo(dst: ArrayBuffer, v: number | bigint): Uint8Array { - const b = BigInt(v); - if (b < 0n) { - throw new Error(`underflow, value is negative: ${v}`); - } - if (b > MAX_U62) { - throw new Error(`overflow, value larger than 62-bits: ${v}`); - } - const n = Number(b); - if (n <= MAX_U6) { - return setUint8(dst, n); - } - if (n <= MAX_U14) { - return setUint16(dst, n | 0x4000); - } - if (n <= MAX_U30) { - return setUint32(dst, n | 0x80000000); - } - return setUint64(dst, b | 0xc000000000000000n); + const lo = split(v); + const buf = new Uint8Array(dst, 0, lengthQuic(parts.hi, lo)); + writeQuic(buf, parts.hi, lo, buf.length); + return buf; } /** - * Encodes a number or bigint as a QUIC variable-length integer. + * Encodes a value as a QUIC variable-length integer. * Returns a new Uint8Array containing the encoded bytes. */ export function encode(v: number | bigint): Uint8Array { @@ -263,41 +101,17 @@ export function encode(v: number | bigint): Uint8Array { * Returns a tuple of [value, remaining buffer]. */ export function decodeBigInt(buf: Uint8Array): [bigint, Uint8Array] { - if (buf.length === 0) { - throw new Error("buffer is empty"); - } - - const size = 1 << ((buf[0] & 0xc0) >> 6); - - if (buf.length < size) { - throw new Error(`buffer too short: need ${size} bytes, have ${buf.length}`); - } - - const view = new DataView(buf.buffer, buf.byteOffset, size); - const remain = buf.subarray(size); - - let value: bigint; - - if (size === 1) { - value = BigInt(buf[0] & 0x3f); - } else if (size === 2) { - value = BigInt(view.getUint16(0) & 0x3fff); - } else if (size === 4) { - value = BigInt(view.getUint32(0) & 0x3fffffff); - } else if (size === 8) { - value = view.getBigUint64(0) & 0x3fffffffffffffffn; - } else { - throw new Error("impossible"); - } - - return [value, remain]; + const size = sizeOf(buf, peekQuic); + const lo = readQuic(buf, 0, size); + return [toBigInt(parts.hi, lo), buf.subarray(size)]; } /** * Decodes a QUIC variable-length integer from a buffer. - * Values above 53 bits lose precision; use {@link decodeBigInt} for exact decoding. + * Throws above `Number.MAX_SAFE_INTEGER` rather than rounding; use {@link decodeBigInt} for those. */ export function decode(buf: Uint8Array): [number, Uint8Array] { - const [value, remain] = decodeBigInt(buf); - return [Number(value), remain]; + const size = sizeOf(buf, peekQuic); + const lo = readQuic(buf, 0, size); + return [toNumber(parts.hi, lo), buf.subarray(size)]; } diff --git a/quest/m1/rs2ts/README.md b/quest/m1/rs2ts/README.md index a075224ac0..e37edf9eb0 100644 --- a/quest/m1/rs2ts/README.md +++ b/quest/m1/rs2ts/README.md @@ -30,21 +30,21 @@ Decided in planning (2026-09-27), with the spike data in and bytes out, no runtime. The async helper methods move behind an `async` cargo feature; rs2ts reads the crate without it and JS reimplements the helpers with Promises. No second crate. -- Varints stay 62-bit on the wire; the spec is not bounded to 2^53. Rust's - `VarInt` newtype carries Encode/Decode and JS gets a matching `VarInt` type - with checked conversion to and from `number`. +- Varints are not bounded to 2^53 on the wire: 62 bits in QUIC form, 64 in + leading-ones form. JS holds any `u64` as a `U64` with checked conversion to + and from `number`; varints are only its wire encoding. - The generated TypeScript is committed and a CI lane regenerates it and fails on drift, so JS contributors and npm publishing never need the nightly toolchain Charon pins. It lives inside js/net and `@moq/net` stays the package. -- The `@moq/net` API may change (disposable handles, `VarInt`) as long as it +- The `@moq/net` API may change (disposable handles, `U64`) as long as it is no worse to use; watch, publish, hang, and the demos update in the same change. - Parity: `just test interop --all`, plus moq-net's own tests translated with the code once they run on a mock clock instead of tokio. - The line lands on `dev`: the Rust refactors break moq-net's published API, and the translator and generated code build on them. Only the additive - [JS VarInt](/quest/m1/rs2ts/js-varint.md) lands on `main`. + JS `U64` (`js/net/src/util/u64.ts`) is on `main`, package-internal. - Hand-written js/net fixes keep landing until the generated path replaces them; it is months out. @@ -55,7 +55,6 @@ js/net it replaces, measured with the [browser benchmarks](/quest/m1/browser-ben ## Required - [VarInt codec](/quest/m1/rs2ts/varint-codec.md) - moq-net encodes through a `VarInt` newtype and a concrete slice-based codec, not generic traits on primitives -- [JS VarInt](/quest/m1/rs2ts/js-varint.md) - js/net has a 62-bit `VarInt` type with checked `number` conversion and no BigInt on the hot path - [rs2ts](/quest/m1/rs2ts/translator.md) - a Charon-based translator emits readable TypeScript for moq-net's lite codec, committed and checked for drift in CI - [Sans-IO moq-net](/quest/m1/rs2ts/sans-io/README.md) - moq-net builds and runs without a runtime; async helpers sit behind an `async` feature - [Mock-clock tests](/quest/m1/rs2ts/mock-clock.md) - moq-net's tests run on the sans-IO clock instead of tokio, so they translate with the code diff --git a/quest/m1/rs2ts/ietf.md b/quest/m1/rs2ts/ietf.md index 7c6ee4fea8..b72b735753 100644 --- a/quest/m1/rs2ts/ietf.md +++ b/quest/m1/rs2ts/ietf.md @@ -9,7 +9,7 @@ the hand-written js/net IETF code (about 8.7k lines) is deleted, with ## Plan Values above 2^53 are legal on the IETF wire (request ids, track aliases); -they stay exact as `VarInt` and only fail where code converts them to +they stay exact as `U64` and only fail where code converts them to `number`. Public API: breaks `@moq/net`; retargets to `dev`. Wire: none. diff --git a/quest/m1/rs2ts/js-varint.md b/quest/m1/rs2ts/js-varint.md deleted file mode 100644 index 0c17bc4432..0000000000 --- a/quest/m1/rs2ts/js-varint.md +++ /dev/null @@ -1,27 +0,0 @@ -# [S] JS VarInt - -## Goal - -js/net has a `VarInt` type that holds the full 62-bit range, converts to and -from `number` with a loud error outside the safe range, and encodes and -decodes without BigInt on the hot path. It is the TypeScript type rs2ts maps -Rust's `VarInt` to. - -## Plan - -Measured in node 24 for an 8-byte encode plus decode: `number` written as two -`u32` halves 2.4 ns, a `{hi, lo}` pair 4.8 ns, `bigint` 10 ns, and js/net -today (BigInt on the wire, then `Number()`) 32 ns. The leading-ones encoder -also converts every value to BigInt, even small ones. - -Guidance: - -- Store two `u32` halves; offer `fromNumber` and `toNumber` (throwing above - 2^53 or on a negative or fractional input), `fromBigInt` and `toBigInt`, - and comparison and increment methods so sequence logic never converts. -- Move js/net's varint reading and writing onto it, dropping the BigInt - round trip for QUIC and leading-ones varints. -- Unit-test the boundaries (2^30, 2^53, 2^62 - 1) against Rust's encoder in - `just test interop`. - -Public API: additive to `@moq/net`; lands on `main`. Wire: none. diff --git a/quest/m1/rs2ts/lite.md b/quest/m1/rs2ts/lite.md index 256dd08783..f42bccd742 100644 --- a/quest/m1/rs2ts/lite.md +++ b/quest/m1/rs2ts/lite.md @@ -12,7 +12,7 @@ first-frame latency are no worse than the hand-written js/net. ## Plan - The `@moq/net` API may change where the generated shape is no worse to - use: disposable handles (`using`), `VarInt` for sequences and ids. Update + use: disposable handles (`using`), `U64` for sequences and ids. Update watch, publish, hang, room, and the demos in the same change, and the `doc/` pages for anything user-facing. - A forgotten `drop()` leaves a track open forever: add a debug-only diff --git a/quest/m1/rs2ts/translator.md b/quest/m1/rs2ts/translator.md index 8459752e54..2f9f222427 100644 --- a/quest/m1/rs2ts/translator.md +++ b/quest/m1/rs2ts/translator.md @@ -29,11 +29,13 @@ Mapping decided in planning: `[Symbol.dispose]`; `Arc`/`Rc` of a type with drop glue become an explicit refcount. JS is single-threaded, so `Mutex` and atomics become plain access. -- Rust `VarInt` maps to the [JS VarInt](/quest/m1/rs2ts/js-varint.md) type. +- Rust `u64` (and `VarInt` while it lasts) maps to js/net's `U64` + (`js/net/src/util/u64.ts`), read and written as a varint by + `Cursor.varint()` and `Writer.varint()`. Integers up to 32 bits and `usize` map to `number` with checked arithmetic that throws on overflow; never wrap silently. A `u64` or `i64` never maps to a lossy `number`: the model accepts `u64::MAX` (e.g. - `model/subscription.rs`), so each one either becomes `VarInt` or an + `model/subscription.rs`), so each one either becomes `U64` or an `Option` in the source, or maps to a full-width 64-bit TypeScript type. Guidance: @@ -56,4 +58,3 @@ translates. Wire: none. ## Required - [VarInt codec](/quest/m1/rs2ts/varint-codec.md) - the codec shape the translator targets -- [JS VarInt](/quest/m1/rs2ts/js-varint.md) - the TypeScript type `VarInt` maps to diff --git a/rs/moq-net/src/test_interop.rs b/rs/moq-net/src/test_interop.rs index 031113d1a2..24d3d03abe 100644 --- a/rs/moq-net/src/test_interop.rs +++ b/rs/moq-net/src/test_interop.rs @@ -1,16 +1,70 @@ -//! Response bytes cross the language boundary; each subscriber runs on a mock transport. +//! Bytes cross the language boundary to a Bun script under `test/interop`: subscription +//! responses for subscribers on a mock transport, and varint encodings. -pub(crate) fn fin(version: &str, started: bool, clean: bool, responses: Vec) -> Vec { - let input = serde_json::json!({ "version": version, "started": started, "clean": clean, "responses": responses }); +use crate::coding::{Decode, Encode, VarInt}; +use crate::ietf; + +// Runs a script with a JSON argument, returning its stdout. +fn bun(script: &str, input: serde_json::Value) -> Vec { let output = std::process::Command::new("bun") - .arg(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/interop/bare-fin.ts")) + .arg(format!("{}/../../test/interop/{script}", env!("CARGO_MANIFEST_DIR"))) .arg(input.to_string()) .output() - .expect("Bun must be installed for the bare-FIN interop test"); + .expect("Bun must be installed for the interop tests"); assert!( output.status.success(), - "JS subscriber failed: {}", + "{script} failed: {}", String::from_utf8_lossy(&output.stderr) ); - serde_json::from_slice(&output.stdout).expect("JS publisher returned response bytes") + output.stdout +} + +pub(crate) fn fin(version: &str, started: bool, clean: bool, responses: Vec) -> Vec { + let input = serde_json::json!({ "version": version, "started": started, "clean": clean, "responses": responses }); + serde_json::from_slice(&bun("bare-fin.ts", input)).expect("JS publisher returned response bytes") +} + +/// Every varint size boundary in both formats, plus the 2^53 edge of a JS `number`. js/net's own +/// tests cover leading-ones values past 2^62 - 1, where moq-net's `VarInt` stops. +#[test] +#[ignore = "requires Bun; run by just test interop"] +fn varint_interop() { + let mut values = vec![0u64]; + for bits in [6, 7, 14, 21, 28, 30, 32, 35, 42, 49, 53, 56] { + values.extend([(1 << bits) - 1, 1 << bits]); + } + values.push(VarInt::MAX.into_inner()); + let values: Vec = values.into_iter().map(|v| VarInt::from_u64(v).unwrap()).collect(); + + let quic = |v: &VarInt| { + let mut buf = Vec::new(); + v.encode_quic(&mut buf).unwrap(); + buf + }; + let leading_ones = |v: &VarInt| { + let mut buf = Vec::new(); + v.encode(&mut buf, ietf::Version::Draft17).unwrap(); + buf + }; + + let input = serde_json::json!({ + "values": values.iter().map(ToString::to_string).collect::>(), + "quic": values.iter().map(quic).collect::>(), + "leadingOnes": values.iter().map(leading_ones).collect::>(), + }); + let output: serde_json::Value = + serde_json::from_slice(&bun("varint.ts", input)).expect("JS returned its encodings"); + let js = |format: &str| -> Vec> { serde_json::from_value(output[format].clone()).unwrap() }; + + for ((value, js_quic), js_leading) in values.iter().zip(js("quic")).zip(js("leadingOnes")) { + assert_eq!(js_quic, quic(value), "QUIC encoding of {value}"); + assert_eq!(js_leading, leading_ones(value), "leading-ones encoding of {value}"); + + let mut buf = js_quic.as_slice(); + assert_eq!(VarInt::decode_quic(&mut buf).unwrap(), *value); + assert!(buf.is_empty()); + let mut buf = js_leading.as_slice(); + assert_eq!(VarInt::decode(&mut buf, ietf::Version::Draft17).unwrap(), *value); + assert!(buf.is_empty()); + } } diff --git a/test/interop/README.md b/test/interop/README.md index 956eaa42af..0a0a5c1d86 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -165,6 +165,8 @@ contract](../README.md). ```text interop.sh orchestrator: build clients, run the relay + matrix or media checks interop.toml relay config (anonymous, self-signed localhost) +bare-fin.ts the JS side of `just test bare-fin`, driven by moq-net's tests +varint.ts the JS side of the varint check, driven by moq-net's tests clients/ python/interop.py publish/subscribe via py/moq-rs (import moq) go/main.go publish/subscribe via go/wrapper (import moq-go/moq) @@ -195,3 +197,12 @@ transport. It checks bare FIN before and after SUBSCRIBE\_START on lite-05/06/07 and FIN without PUBLISH\_DONE on IETF draft-19. Clean-end controls use the same path. This tests response interoperability, not network delivery or relay behavior. The interop workflow runs it alongside the real-transport matrix. + +## Varints + +Every `just test interop` run starts with `varint_interop` in moq-net, which +hands moq-net's QUIC and leading-ones encodings of each varint size boundary +(plus 2^53, where a JS `number` stops being exact, and 2^62 - 1) to +`varint.ts`. That script decodes them into js/net's `U64`, checks its +`number` conversion, and returns js/net's own encodings, which Rust requires to +match byte for byte and decode back to the same value. diff --git a/test/interop/varint.ts b/test/interop/varint.ts new file mode 100644 index 0000000000..464a34b5f0 --- /dev/null +++ b/test/interop/varint.ts @@ -0,0 +1,43 @@ +/** Decode Rust's varint encodings into js/net's U64, then hand back js/net's own encodings. */ +import assert from "node:assert/strict"; +import { Version } from "../../js/net/src/ietf/version.ts"; +import { Reader, Writer } from "../../js/net/src/stream.ts"; +import { U64 } from "../../js/net/src/util/u64.ts"; + +// Each value is a decimal string, since JSON numbers round past 2^53. +const input: { values: string[]; quic: number[][]; leadingOnes: number[][] } = JSON.parse(process.argv[2]); + +async function encode(v: U64, version?: Version): Promise { + const bytes: number[] = []; + const writer = new Writer(new WritableStream({ write: (chunk) => void bytes.push(...chunk) }), version); + await writer.varint(v); + writer.close(); + await writer.closed; + return bytes; +} + +async function decode(bytes: number[], version?: Version): Promise { + const reader = new Reader(undefined, new Uint8Array(bytes), version); + const v = await reader.varint(); + assert(await reader.done(), `trailing bytes after ${v}`); + return v; +} + +const output: { quic: number[][]; leadingOnes: number[][] } = { quic: [], leadingOnes: [] }; +for (const [i, value] of input.values.entries()) { + const expected = U64.fromBigInt(BigInt(value)); + for (const [format, version] of [ + ["quic", undefined], + ["leadingOnes", Version.DRAFT_17], + ] as const) { + const decoded = await decode(input[format][i], version); + assert(decoded.equals(expected), `${format}: decoded ${decoded}, expected ${value}`); + assert.equal(decoded.toString(), value); + if (BigInt(value) <= BigInt(Number.MAX_SAFE_INTEGER)) assert.equal(decoded.toNumber(), Number(value)); + else assert.throws(() => decoded.toNumber(), RangeError); + output[format].push(await encode(expected, version)); + } +} + +// Stdout is the encoding channel back to Rust. +console.log(JSON.stringify(output)); diff --git a/test/justfile b/test/justfile index 1c88dcb677..ee082ed231 100644 --- a/test/justfile +++ b/test/justfile @@ -38,9 +38,11 @@ harness: # client, and the GStreamer moqsrc plugin). --timeout 30 gives headless Chromium # cold-start headroom. Other flags pass through, e.g. # `just test interop --publishers rust,python --subscribers rust,c`. +# Both runs first check js/net's varints against moq-net's encoder at every size boundary. interop *args: #!/usr/bin/env bash set -euo pipefail + just rs test -p moq-net --lib varint_interop --run-ignored all if [[ "${1:-}" == --all ]]; then shift ./interop/interop.sh --publishers rust,python,go,js --subscribers rust,python,go,js,js-native-node,js-native-bun,c,gst --timeout 30 "$@"