Skip to content

Commit f8bacd0

Browse files
committed
feat(scripts): a third test-budget class, "compiles the package type graph once"
TEST_BUDGET_CLASSES gains its third value, as ruling record 6212133431 (letter A) names it. The number rule and the three prohibitions of ruling record 6181973363 are unchanged. The refusal now lists every class ("one of ...") and says "none of the 3 classes the rulings name" instead of "neither class". The self-test pins the three spellings as literals and reads the third end to end: the ruling's line under the spec files' header, the two earlier classes in that same header, six near misses refused naming all three classes, and a --diff run from disk that holds the file at 18.10 s of its 30 s and lists it at 31 s. Battery pin 14 -> 18. Claude-Session: https://claude.ai/code/session_016PVjjuVPte5XEJvaQx1E4t Co-authored-by: Claude <noreply@anthropic.com>
1 parent 497518c commit f8bacd0

1 file changed

Lines changed: 107 additions & 16 deletions

File tree

‎scripts/measure-test-shard-timings.mjs‎

Lines changed: 107 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1222,21 +1222,25 @@ export function readWindowCaptures(dir) {
12221222
return { byRun, problems, present: true };
12231223
}
12241224

1225-
// -- A test file's own measured budget (ruling record 6181973363, letter B) --
1225+
// -- A test file's own measured budget (ruling records 6181973363, 6212133431) --
12261226
//
1227-
// Two classes of test file cannot fit the 10 s line without being weakened:
1227+
// Three classes of test file cannot fit the 10 s line without being weakened:
12281228
// one that spawns the real producer (a gate, a generator) once per
1229-
// single-cause fixture, and one whose subject is every module of its package.
1230-
// A file of either class carries its own budget in place of the line, as ONE
1231-
// line in its header, and is listed only when the window reads it above it:
1229+
// single-cause fixture, and one whose subject is every module of its package
1230+
// (both ruling record 6181973363, letter B); and one whose floor is a single
1231+
// compile of its package's type graph, every probe after the first reading
1232+
// that one program (ruling record 6212133431, letter A, which adds this class
1233+
// and changes nothing else). A file of any class carries its own budget in
1234+
// place of the line, as ONE line in its header, and is listed only when the
1235+
// window reads it above it:
12321236
//
12331237
// // test-budget: SECONDS — CLASS: REASON
12341238
//
12351239
// SECONDS the measured CI reading x CEILING_HEADROOM, rounded UP to the
12361240
// next 10 s: a whole number, a multiple of 10, above
12371241
// FILE_BUDGET_SECONDS (one at or under the line lifts nothing). It
12381242
// rises only by a PR that re-measures and says why in REASON.
1239-
// CLASS exactly one of TEST_BUDGET_CLASSES, spelled as the ruling names it.
1243+
// CLASS exactly one of TEST_BUDGET_CLASSES, spelled as its ruling names it.
12401244
// REASON why the file is of that class; not empty.
12411245
//
12421246
// The separators are an em dash between single spaces, and a colon and a space
@@ -1255,6 +1259,7 @@ export function readWindowCaptures(dir) {
12551259
export const TEST_BUDGET_CLASSES = Object.freeze([
12561260
'spawns the real producer once per single-cause fixture',
12571261
'whole-package import',
1262+
'compiles the package type graph once',
12581263
]);
12591264
const TEST_BUDGET_ATTEMPT = /^(?:\/\/+|\/\*+|\*)\s*test[-_ ]?budget\s*:/i;
12601265
const TEST_BUDGET_LINE = /^\/\/ test-budget: (\S+) — ([^:]+): (.*)$/u;
@@ -1265,8 +1270,8 @@ export function readTestBudget(text, label) {
12651270
const refuse = (line, why) => {
12661271
throw new Error(
12671272
`${label}:${line}: ${why}. A test file's budget is ONE header line, \`// test-budget: SECONDS — CLASS: REASON\` ` +
1268-
`(ruling record 6181973363): SECONDS is the measured CI reading x ${CEILING_HEADROOM} rounded up to the next 10 s, ` +
1269-
`above the ${FILE_BUDGET_SECONDS} s line; CLASS is ${TEST_BUDGET_CLASSES.map((c) => `"${c}"`).join(' or ')}; ` +
1273+
`(ruling records 6181973363 and 6212133431): SECONDS is the measured CI reading x ${CEILING_HEADROOM} rounded up to the next 10 s, ` +
1274+
`above the ${FILE_BUDGET_SECONDS} s line; CLASS is one of ${TEST_BUDGET_CLASSES.map((c) => `"${c}"`).join(', ')}; ` +
12701275
'REASON is not empty. Refusing to guess which budget the file is held to.'
12711276
);
12721277
};
@@ -1292,7 +1297,7 @@ export function readTestBudget(text, label) {
12921297
const seconds = Number(number);
12931298
if (seconds % 10 !== 0) refuse(line, `the budget ${seconds} s is not on the 10 s grid`);
12941299
if (seconds <= FILE_BUDGET_SECONDS) refuse(line, `the budget ${seconds} s does not lift the ${FILE_BUDGET_SECONDS} s line`);
1295-
if (!TEST_BUDGET_CLASSES.includes(cls)) refuse(line, `the class "${cls}" is neither class the ruling names`);
1300+
if (!TEST_BUDGET_CLASSES.includes(cls)) refuse(line, `the class "${cls}" is none of the ${TEST_BUDGET_CLASSES.length} classes the rulings name`);
12961301
if (reason.trim() === '') refuse(line, 'the reason is empty');
12971302
found = { seconds, cls, reason: reason.trim(), line };
12981303
}
@@ -1660,8 +1665,9 @@ const SELF_TEST_BATTERIES = Object.freeze({
16601665
// (the density-only and the code fixture first), and the per-file budget.
16611666
'refresh diff and per-file budget (#22949 E)': 22,
16621667
// Ruling record 6181973363: a test file's own measured budget, the
1663-
// declaration's grammar and its refusals, and the list held to it.
1664-
'per-file budget declarations (ruling 6181973363)': 14,
1668+
// declaration's grammar and its refusals, and the list held to it; ruling
1669+
// record 6212133431: its third class, read in memory and from disk.
1670+
'per-file budget declarations (ruling 6181973363)': 18,
16651671
});
16661672

16671673
// DELETING an entry silences that battery's floor exactly as effectively as
@@ -2979,11 +2985,20 @@ function selfTest() {
29792985

29802986
// -- A test file's own measured budget (ruling record 6181973363) ----------
29812987
battery('per-file budget declarations (ruling 6181973363)');
2982-
const [producerClass, wholePackageClass] = TEST_BUDGET_CLASSES;
2988+
const [producerClass, wholePackageClass, typeGraphClass] = TEST_BUDGET_CLASSES;
2989+
// The ruled spellings, as LITERALS: a case that read them back out of
2990+
// TEST_BUDGET_CLASSES would still pass with a value deleted from it.
2991+
const ruledClasses = ['spawns the real producer once per single-cause fixture', 'whole-package import', 'compiles the package type graph once'];
29832992
const declare = (seconds, cls = producerClass, reason = 'each case spawns the gate once') =>
29842993
`// test-budget: ${seconds} — ${cls}: ${reason}`;
29852994
// Line 3 of the file is `decl`; `rest` follows the header's blank line 4.
29862995
const fileWith = (decl, rest = "import { it } from 'vitest';\n") => `// Copyright (c) 2026 ObjectStack.\n//\n${decl}\n\n${rest}`;
2996+
// The header of the two spec files the third class was ruled for, with the
2997+
// declaration on line 2, where every declaration in the tree sits.
2998+
const specFileWith = (decl) =>
2999+
`// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.\n${decl}\n\nimport { describe, it, expect } from 'vitest';\nimport ts from 'typescript';\n`;
3000+
const typeGraphLine =
3001+
'// test-budget: 30 — compiles the package type graph once: one ts.createProgram over the published entries, and every probe after the first reads that one program';
29873002
const refusalOf = (fn) => {
29883003
try {
29893004
fn();
@@ -2993,11 +3008,49 @@ function selfTest() {
29933008
return null;
29943009
};
29953010
check(() => {
2996-
// The two classes, spelled as the ruling names them, and no third.
2997-
if (TEST_BUDGET_CLASSES.length !== 2 || producerClass !== 'spawns the real producer once per single-cause fixture' || wholePackageClass !== 'whole-package import') {
3011+
// The three classes, spelled as their rulings name them, in order, and no fourth.
3012+
if (
3013+
TEST_BUDGET_CLASSES.length !== 3 ||
3014+
producerClass !== ruledClasses[0] ||
3015+
wholePackageClass !== ruledClasses[1] ||
3016+
typeGraphClass !== ruledClasses[2]
3017+
) {
29983018
throw new Error(`budget: the classes read ${JSON.stringify(TEST_BUDGET_CLASSES)}`);
29993019
}
30003020
});
3021+
check(() => {
3022+
// The third class (ruling record 6212133431), in the line the ruling
3023+
// writes, under the header of the files it was ruled for: a declared budget.
3024+
const d = readTestBudget(specFileWith(typeGraphLine), 'packages/spec/src/contracts/scoped-context.test.ts');
3025+
if (d?.seconds !== 30 || d.cls !== ruledClasses[2] || d.line !== 2 || !d.reason.startsWith('one ts.createProgram over')) {
3026+
throw new Error(`budget: the third class's declaration read ${JSON.stringify(d)}`);
3027+
}
3028+
});
3029+
check(() => {
3030+
// The two earlier classes still read, in that same header: the third
3031+
// took nothing from them.
3032+
for (const cls of ruledClasses.slice(0, 2)) {
3033+
const d = readTestBudget(specFileWith(declare(40, cls, 'r')), 'f.test.ts');
3034+
if (d?.seconds !== 40 || d.cls !== cls || d.line !== 2) throw new Error(`budget: the class "${cls}" read ${JSON.stringify(d)}`);
3035+
}
3036+
});
3037+
check(() => {
3038+
// A near miss of the third class is REFUSED, never read as a nearby class:
3039+
// the refusal names the file and line, the spelling, and all three classes.
3040+
for (const near of [
3041+
'compiles the package type-graph once',
3042+
'compiles the package type graph',
3043+
'compiles the type graph once',
3044+
'compiles the packages type graph once',
3045+
'Compiles the package type graph once',
3046+
'compiles the package type graph once ',
3047+
]) {
3048+
const msg = refusalOf(() => readTestBudget(fileWith(declare(30, near)), 'f.test.ts'));
3049+
if (!msg?.startsWith('f.test.ts:3: ') || !msg.includes(`"${near}"`) || !ruledClasses.every((c) => msg.includes(`"${c}"`))) {
3050+
throw new Error(`budget: the near miss "${near}" read as ${msg}`);
3051+
}
3052+
}
3053+
});
30013054
check(() => {
30023055
// Each class parses, in a `//` header and below a header docblock.
30033056
const d = readTestBudget(fileWith(declare(1130)), 'f.test.ts');
@@ -3016,9 +3069,9 @@ function selfTest() {
30163069
}
30173070
});
30183071
check(() => {
3019-
// A class naming neither is REFUSED, naming the file and the line.
3072+
// A class naming none of them is REFUSED, naming the file and the line.
30203073
const msg = refusalOf(() => readTestBudget(fileWith(declare(1130, 'slow by nature')), 'packages/x/a.test.ts'));
3021-
if (!msg?.startsWith('packages/x/a.test.ts:3: ') || !msg.includes('slow by nature')) throw new Error(`budget: a class naming neither read as ${msg}`);
3074+
if (!msg?.startsWith('packages/x/a.test.ts:3: ') || !msg.includes('slow by nature')) throw new Error(`budget: a class naming none of them read as ${msg}`);
30223075
});
30233076
check(() => {
30243077
// A number off the 10 s grid, at or under the line, or not a whole number is REFUSED.
@@ -3154,6 +3207,44 @@ function selfTest() {
31543207
rmSync(dir, { recursive: true, force: true });
31553208
}
31563209
});
3210+
check(() => {
3211+
// The third class end to end, from a file on disk through --diff: at the
3212+
// card's slowest CI reading (18.1 s) the file is held to its 30 s and named
3213+
// with its class; read over the 30 s, it is listed against them.
3214+
const dir = mkdtempSync(path.join(tmpdir(), 'os-refresh-type-graph-'));
3215+
try {
3216+
const pkgDir = path.join(dir, 'spec');
3217+
mkdirSync(path.join(pkgDir, 'src', 'integration'), { recursive: true });
3218+
writeFileSync(path.join(pkgDir, 'src', 'integration', 'graph.test.ts'), specFileWith(typeGraphLine));
3219+
writeFileSync(path.join(dir, 'prior.json'), JSON.stringify(priorWindow));
3220+
writeFileSync(path.join(dir, 'refreshed.json'), JSON.stringify(codeWindow));
3221+
const run = (ms, out) => {
3222+
const caps = path.join(dir, `captures-${out}`);
3223+
mkdirSync(path.join(caps, 'n1', '1'), { recursive: true });
3224+
writeFileSync(path.join(caps, 'n1', '1', 'shard-1.json'), JSON.stringify(capture([fileRow('spec', 'src/integration/graph.test.ts', ms)])));
3225+
const argv = ['--diff', '--prior', path.join(dir, 'prior.json'), '--refreshed', path.join(dir, 'refreshed.json'), '--captures', caps, '--out', path.join(dir, out)];
3226+
const quiet = console.error;
3227+
console.error = () => {};
3228+
try {
3229+
return diffMain(argv, { packageDir: (name) => (name === 'spec' ? pkgDir : null) });
3230+
} finally {
3231+
console.error = quiet;
3232+
}
3233+
};
3234+
const heldCode = run(18_100, 'held.md');
3235+
const held = readFileSync(path.join(dir, 'held.md'), 'utf8');
3236+
if (heldCode !== 0 || held.includes('| `src/integration/graph.test.ts`') || !held.includes('`src/integration/graph.test.ts` (`spec`) 18.10 s of its 30 s, compiles the package type graph once')) {
3237+
throw new Error(`cli: a type-graph file under its declared budget exited ${heldCode} and wrote:\n${held}`);
3238+
}
3239+
const overCode = run(31_000, 'over.md');
3240+
const over = readFileSync(path.join(dir, 'over.md'), 'utf8');
3241+
if (overCode !== 0 || !over.includes('| `src/integration/graph.test.ts` | `spec` | `n1` | 31.00 (declared 30) | 1 / 1 |')) {
3242+
throw new Error(`cli: a type-graph file over its declared budget exited ${overCode} and wrote:\n${over}`);
3243+
}
3244+
} finally {
3245+
rmSync(dir, { recursive: true, force: true });
3246+
}
3247+
});
31573248

31583249
// -- The floor: every declared battery RAN, and ran its cases (#13489) ----
31593250
//

0 commit comments

Comments
 (0)