feat(net): BigInt-free varint codec, internal U64, and checked Varint.decode

.github/workflows/nightly.yml

@@ -92,6 +92,12 @@ jobs:
         if: ${{ !cancelled() }}
         run: nix develop --command bun js/net/bench/frames.ts
 
+      # Times one varint encode and decode per wire format. No threshold: it only
+      # has to keep running, and its numbers are the baseline for the generated codec.
+      - name: JS varint benchmark
+        if: ${{ !cancelled() }}
+        run: nix develop --command bun js/net/bench/varint.ts
+
       # Fails if publishing a group costs more as the track retains more groups,
       # which means the latency guard or the cache went back to scanning them all.
       - name: JS track retention benchmark

js/hang/src/container/consumer.test.ts

@@ -1652,3 +1652,8 @@ for (const end of [
 		}
 	});
 }
+
+test("LegacyFormat rejects a timestamp past 2^53 - 1 instead of rounding", () => {
+	const frame = Varint.encode(2n ** 53n + 1n);
+	expect(() => new LegacyFormat("video").decode(frame)).toThrow(/larger than 53-bits/);
+});

js/loc/src/index.test.ts

@@ -133,3 +133,19 @@ test("Format decodes the draft-03 timestamp property", () => {
 	expect(decoded.timestamp).toBe(4242 as Time.Micro);
 	expect(decoded.payload).toEqual(payload);
 });
+
+test("Format skips an unknown property whose value needs all 62 bits", () => {
+	const props = concat(
+		Varint.encode(0x02),
+		Varint.encode(2n ** 62n - 1n),
+		Varint.encode(PROP_TIMESTAMP - 0x02),
+		Varint.encode(1_000),
+	);
+	const [decoded] = new Format().decode(buildFrame(props, new Uint8Array([1])));
+	expect(decoded.timestamp).toBe(1_000 as Time.Micro);
+});
+
+test("Format rejects a timestamp past 2^53 - 1 instead of rounding", () => {
+	const props = concat(Varint.encode(PROP_TIMESTAMP), Varint.encode(2n ** 53n));
+	expect(() => new Format().decode(buildFrame(props, new Uint8Array()))).toThrow(/larger than 53-bits/);
+});

js/loc/src/index.ts

@@ -73,17 +73,18 @@ export class Format {
 			prevType = abs;
 			cursor = afterDelta;
 
-			if (abs % 2 === 0) {
-				const [value, afterValue] = Moq.Varint.decode(cursor);
-				cursor = afterValue;
-				if (abs === PROP_TIMESTAMP || abs === PROP_TIMESTAMP_DRAFT03) {
-					timestamp = value;
-				} else if (abs === PROP_TIMESCALE) {
-					if (value === 0) {
-						throw new Error("loc: timescale property must be non-zero");
-					}
-					timescale = value;
+			if (abs === PROP_TIMESTAMP || abs === PROP_TIMESTAMP_DRAFT03) {
+				[timestamp, cursor] = Moq.Varint.decode(cursor);
+			} else if (abs === PROP_TIMESCALE) {
+				let value: number;
+				[value, cursor] = Moq.Varint.decode(cursor);
+				if (value === 0) {
+					throw new Error("loc: timescale property must be non-zero");
 				}
+				timescale = value;
+			} else if (abs % 2 === 0) {
+				// An unknown varint property may use all 62 bits, which a number can't hold; skip it.
+				cursor = Moq.Varint.decodeBigInt(cursor)[1];
 			} else {
 				const [len, afterLenInner] = Moq.Varint.decode(cursor);
 				if (afterLenInner.byteLength < len) {

js/net/bench/varint.ts

@@ -0,0 +1,89 @@
+/** Time one varint encode and one decode, for both wire formats, at their 1, 2, 4, and 8-byte sizes. */
+import { Version } from "../src/ietf/version.ts";
+import { Cursor } from "../src/stream.ts";
+import * as Varint from "../src/varint.ts";
+
+// Decodes run through one Cursor over this many copies, so the Cursor itself isn't timed.
+const run = 1024;
+const runs = 2_000;
+const reps = 9;
+const scratch = new ArrayBuffer(9);
+let checksum = 0;
+
+// QUIC varints (lite, and moq-transport before draft-17), and leading-ones (draft-17+), each at the
+// largest value of its 1, 2, 4, and 8-byte forms that a `number` holds.
+const formats = [
+	{
+		name: "quic",
+		version: undefined,
+		encode: Varint.encodeTo,
+		values: [2 ** 6 - 1, 2 ** 14 - 1, 2 ** 30 - 1, Number.MAX_SAFE_INTEGER],
+	},
+	{
+		name: "leading-ones",
+		version: Version.DRAFT_17,
+		encode: Varint.encodeLeadingOnesTo,
+		values: [2 ** 7 - 1, 2 ** 14 - 1, 2 ** 28 - 1, Number.MAX_SAFE_INTEGER],
+	},
+];
+
+interface Case {
+	name: string;
+	ops: [string, () => void][];
+}
+
+const cases: Case[] = [];
+for (const format of formats) {
+	for (const value of format.values) {
+		const one = format.encode(scratch, value).slice();
+		const encoded = new Uint8Array(one.byteLength * run);
+		for (let i = 0; i < run; i++) encoded.set(one, i * one.byteLength);
+
+		cases.push({
+			name: `${format.name},${one.byteLength}-byte`,
+			ops: [
+				[
+					"encode",
+					() => {
+						for (let i = 0; i < run; i++) checksum += format.encode(scratch, value).byteLength;
+					},
+				],
+				[
+					"decode",
+					() => {
+						const cursor = new Cursor(encoded, format.version);
+						for (let i = 0; i < run; i++) checksum += cursor.u53();
+					},
+				],
+				// The same, as the U64 the generated codec will use, which allocates.
+				[
+					"decode-varint",
+					() => {
+						const cursor = new Cursor(encoded, format.version);
+						for (let i = 0; i < run; i++) checksum += cursor.varint().lo;
+					},
+				],
+			],
+		});
+	}
+}
+
+// Nanoseconds per varint for the fastest of several reps, since a slower one measured the machine.
+function time(body: () => void): number {
+	let best = Number.POSITIVE_INFINITY;
+	for (let rep = 0; rep < reps; rep++) {
+		const start = performance.now();
+		for (let i = 0; i < runs; i++) body();
+		best = Math.min(best, ((performance.now() - start) * 1e6) / (runs * run));
+	}
+	return best;
+}
+
+// Run every case once first, so the JIT has seen all of them and the first row isn't timing warmup.
+for (const { ops } of cases) for (const [, body] of ops) for (let i = 0; i < runs; i++) body();
+
+console.log("format,value,op,ns_per_op");
+for (const { name, ops } of cases) {
+	for (const [op, body] of ops) console.log(`${name},${op},${time(body).toFixed(1)}`);
+}
+if (checksum === 0) throw new Error("benchmark did no work");

js/net/src/index.ts

@@ -28,5 +28,5 @@ export * as Path from "./path.ts";
 export * as Time from "./time.ts";
 /** Track role handles. */
 export * as Track from "./track.ts";
-/** QUIC variable-length integer encoding and decoding. */
+/** Varint encoding and decoding, in QUIC's format and moq-transport's leading-ones format. */
 export * as Varint from "./varint.ts";

js/net/src/stream.test.ts

@@ -13,6 +13,7 @@ import {
 import { Version } from "./ietf/version.ts";
 import { type Cursor, Reader, Stream, Writer } from "./stream.ts";
 import { TimeoutError } from "./util/timeout.ts";
+import { U64 } from "./util/u64.ts";
 
 // Helper to create a writable stream that captures written data
 function createTestWritableStream(): { stream: WritableStream<Uint8Array>; written: Uint8Array[] } {
@@ -796,3 +797,55 @@ for (const [version, tooFarBehind] of [
 		if (version !== undefined) expect((err as StreamError).message).toContain("70");
 	});
 }
+
+test("Writer and Reader varint round-trip every size in both formats", async () => {
+	const values = [0n, 63n, 64n, 127n, 128n, 2n ** 14n, 2n ** 21n, 2n ** 28n, 2n ** 30n, 2n ** 32n, 2n ** 35n];
+	values.push(2n ** 42n, 2n ** 49n, 2n ** 53n, 2n ** 56n, 2n ** 62n - 1n);
+	for (const version of [undefined, Version.DRAFT_16, Version.DRAFT_17, Version.DRAFT_19]) {
+		// Only leading-ones varints reach past 62 bits.
+		const all =
+			version === Version.DRAFT_17 || version === Version.DRAFT_19
+				? [...values, 2n ** 62n, 2n ** 64n - 1n]
+				: values;
+		const { stream, written } = createTestWritableStream();
+		const writer = new Writer(stream, version);
+		for (const value of all) await writer.varint(U64.fromBigInt(value));
+		writer.close();
+		await writer.closed;
+
+		const reader = new Reader(undefined, concatChunks(written), version);
+		for (const value of all) expect((await reader.varint()).toBigInt()).toBe(value);
+		expect(await reader.done()).toBe(true);
+	}
+});
+
+test("Writer varint refuses a QUIC varint past 62 bits before emitting bytes", async () => {
+	const { stream, written } = createTestWritableStream();
+	const writer = new Writer(stream);
+	await expect(writer.varint(U64.fromBigInt(2n ** 62n))).rejects.toThrow(/larger than 62-bits/);
+	await expect(writer.varint(U64.MAX)).rejects.toThrow(/larger than 62-bits/);
+	expect(written).toEqual([]);
+});
+
+test("Reader u53 decodes every size in both formats, one byte per chunk", async () => {
+	const values = [0, 63, 64, 127, 128, 16383, 16384, 2 ** 21, 2 ** 28 - 1, 2 ** 28, 2 ** 30 - 1, 2 ** 30];
+	values.push(2 ** 35, 2 ** 42, 2 ** 49, Number.MAX_SAFE_INTEGER);
+	for (const version of [undefined, Version.DRAFT_17]) {
+		const { stream, written } = createTestWritableStream();
+		const writer = new Writer(stream, version);
+		for (const value of values) await writer.u53(value);
+		writer.close();
+		await writer.closed;
+
+		const bytes = concatChunks(written);
+		const chunked = new ReadableStream<Uint8Array>({
+			start(controller) {
+				for (const byte of bytes) controller.enqueue(new Uint8Array([byte]));
+				controller.close();
+			},
+		});
+		const reader = new Reader(chunked, undefined, version);
+		for (const value of values) expect(await reader.u53()).toBe(value);
+		expect(await reader.done()).toBe(true);
+	}
+});