Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/upstream-pins.json
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@
"cited_by": [
"src/schema-version.ts"
],
"access_note": "Comfy-Org/cloud is PRIVATE. `npm run check:pins -- --verify-remote` resolves this pin only with a token that has access; without one the GitHub API answers 404, which this gate treats as definitive evidence ABOUT THE PIN (exit 1) rather than about the request (exit 2). That classification was deliberately introduced by #66 and is right for a public repo; it mis-fires here. Read a 404 on this pin as 'no access', not 'pin broken', until the gate learns to tell them apart.",
"access_note": "Comfy-Org/cloud is PRIVATE. `npm run check:pins -- --verify-remote` resolves this pin only with a token that has Contents read access. GitHub fine-grained permissions separate Metadata read from Contents read, and inaccessible private resources may return 404. Therefore an ambiguous commit/path 404 triggers a repository metadata lookup, but even matching metadata with `private: true` remains INCONCLUSIVE (exit 2): metadata visibility does not prove Contents access. Only matching metadata with explicit `private: false` makes an object 404 a definite violation (exit 1), because public contents need no permission. Missing, nonboolean, malformed, inaccessible, or indeterminate visibility is also INCONCLUSIVE. This safe limitation means a genuinely missing private object reported as 404 remains inconclusive; successful private commit/content probes still pass. It preserves #66's historical rationale for treating genuine object-scoped 422/451 responses as failures without falsely declaring a private pin broken when endpoint permission was never established.",
"upstream_head_observed": {
"date": "2026-08-21",
"ref": "main",
Expand Down
65 changes: 56 additions & 9 deletions scripts/check-pins.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,10 @@
*
* Exit codes: 0 pinned, 1 violation, 2 inconclusive (registry missing or
* unparseable, nothing scanned, or — in remote mode — no definitive answer from
* upstream). "No definitive answer" includes an unusable token and a rate limit:
* only 404/422/451 are evidence about a pin.
* upstream). "No definitive answer" includes an unusable token, a rate limit,
* and a 404 unless repository metadata explicitly establishes that the
* repository is public. Metadata access to a private repository does not prove
* Contents permission. 422/451 remain object-scoped answers by themselves.
*/

import { spawnSync } from "node:child_process";
Expand Down Expand Up @@ -328,7 +330,10 @@ if (ghProbe.error || ghProbe.status !== 0) {
* Statuses that are evidence about the OBJECT that was asked for. Only these
* may be read as "this pin is broken".
*
* 404 — no such commit/path in this repository
* 404 — no such commit/path, but only after repository metadata identifies
* the matching repository and explicitly says it is public. Public
* contents need no endpoint permission; private metadata visibility
* does not establish private Contents permission.
* 422 — the SHA is well-formed but names no object here
* 451 — the object exists but is legally unavailable; not a pin problem to
* fix by re-pinning, but it is a definite answer about this object
Expand Down Expand Up @@ -358,15 +363,57 @@ function gh(endpoint) {
};
}

function requireDefinitive(result, what) {
if (result.ok || result.definitive) return;
function establishPublicRepository(slug) {
const repository = gh(`repos/${slug}`);
if (!repository.ok) {
const status = repository.http === null ? "no HTTP status (DNS/TLS/proxy)" : `HTTP ${repository.http}`;
inconclusive(
`could not establish public repository visibility for ${slug} — ${status}`,
repository.stderr.split("\n")[0] || "no error output",
"GitHub hides inaccessible private repositories behind 404, so this response cannot prove",
"whether the repository is hidden or the requested commit/path is genuinely absent.",
`Offline result: ${errors.length === 0 ? "clean" : `${errors.length} violation(s)`}`,
);
}

let metadata;
try {
metadata = JSON.parse(repository.stdout);
} catch {
inconclusive(`repository metadata for ${slug} was malformed, so repository visibility was not established`);
}
if (
!metadata ||
typeof metadata !== "object" ||
typeof metadata.full_name !== "string" ||
metadata.full_name.toLowerCase() !== slug.toLowerCase() ||
typeof metadata.private !== "boolean"
) {
inconclusive(`repository metadata for ${slug} was malformed, so repository visibility was not established`);
}
if (metadata.private) {
inconclusive(
`repository metadata for ${slug} identifies a private repository, but metadata visibility does not prove Contents access`,
"GitHub fine-grained permissions separate Metadata read from Contents read. A private-object",
"404 may therefore mean either a missing object or missing endpoint permission.",
);
}
}

function requireDefinitive(result, what, slug) {
if (result.ok) return;
if (result.http === 404) {
establishPublicRepository(slug);
return;
}
if (result.definitive) return;
const status = result.http === null ? "no HTTP status (DNS/TLS/proxy)" : `HTTP ${result.http}`;
inconclusive(
`could not get a definitive answer from GitHub while checking ${what} — ${status}`,
result.stderr.split("\n")[0] || "no error output",
"This is NOT a passing pin and NOT a failing pin — the check did not happen.",
"Only 404/422/451 are answers about the object; 401/403/429/5xx are answers about the",
"request (credentials, rate limit, outage) and prove nothing about the pin.",
"Only 422/451, or 404 after public repository visibility is established, are answers about the",
"object; 401/403/429/5xx are about the request and prove nothing about the pin.",
`Offline result: ${errors.length === 0 ? "clean" : `${errors.length} violation(s)`}`,
);
}
Expand All @@ -393,7 +440,7 @@ for (const [id, pin] of Object.entries(pins)) {
const slug = pin.repo.replace(/^https:\/\/github\.com\//, "").replace(/\.git$/, "");

const commit = gh(`repos/${slug}/commits/${pin.commit}`);
requireDefinitive(commit, `${id} commit ${pin.commit}`);
requireDefinitive(commit, `${id} commit ${pin.commit}`, slug);
if (!commit.ok) {
errors.push(
`${id}: commit ${pin.commit} no longer resolves in ${slug} — ${commit.stderr.split("\n")[0]}`,
Expand All @@ -402,7 +449,7 @@ for (const [id, pin] of Object.entries(pins)) {
}

const contents = gh(`repos/${slug}/contents/${pin.path}?ref=${pin.commit}`);
requireDefinitive(contents, `${id} path ${pin.path}`);
requireDefinitive(contents, `${id} path ${pin.path}`, slug);
if (!contents.ok) {
errors.push(`${id}: ${pin.path} is absent at ${pin.commit} in ${slug}`);
continue;
Expand Down
180 changes: 180 additions & 0 deletions test/upstream-pins.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,186 @@ describe("FC-10 — upstream citations are pinned by SHA, not by branch", () =>
}
});

it.each([
{
name: "hidden private repository commit 404",
commitStatus: 404,
metadataStatus: 404,
expectedStatus: 2,
expectedMessage: "could not establish public repository visibility for example/upstream — HTTP 404",
endpoints: ["api rate_limit", "api repos/example/upstream/commits/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "api repos/example/upstream"],
},
{
name: "public repository commit 404",
commitStatus: 404,
metadataStatus: 200,
metadataPrivate: false,
expectedStatus: 1,
expectedMessage: "commit aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa no longer resolves in example/upstream",
endpoints: ["api rate_limit", "api repos/example/upstream/commits/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "api repos/example/upstream"],
},
{
name: "public repository path 404",
commitStatus: 200,
contentsStatus: 404,
metadataStatus: 200,
metadataPrivate: false,
expectedStatus: 1,
expectedMessage: "README.md is absent at aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa in example/upstream",
endpoints: ["api rate_limit", "api repos/example/upstream/commits/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "api repos/example/upstream/contents/README.md?ref=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "api repos/example/upstream"],
},
{
name: "private repository commit 404 despite readable metadata",
commitStatus: 404,
metadataStatus: 200,
metadataPrivate: true,
expectedStatus: 2,
expectedMessage: "repository metadata for example/upstream identifies a private repository, but metadata visibility does not prove Contents access",
endpoints: ["api rate_limit", "api repos/example/upstream/commits/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "api repos/example/upstream"],
},
{
name: "private repository path 404 despite readable metadata",
commitStatus: 200,
contentsStatus: 404,
metadataStatus: 200,
metadataPrivate: true,
expectedStatus: 2,
expectedMessage: "repository metadata for example/upstream identifies a private repository, but metadata visibility does not prove Contents access",
endpoints: ["api rate_limit", "api repos/example/upstream/commits/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "api repos/example/upstream/contents/README.md?ref=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "api repos/example/upstream"],
},
{
name: "commit 422 remains object-scoped",
commitStatus: 422,
metadataStatus: 200,
expectedStatus: 1,
expectedMessage: "commit aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa no longer resolves in example/upstream",
endpoints: ["api rate_limit", "api repos/example/upstream/commits/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"],
},
{
name: "commit 451 remains object-scoped",
commitStatus: 451,
metadataStatus: 200,
expectedStatus: 1,
expectedMessage: "commit aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa no longer resolves in example/upstream",
endpoints: ["api rate_limit", "api repos/example/upstream/commits/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"],
},
{
name: "repository metadata rejects authentication",
commitStatus: 404,
metadataStatus: 403,
expectedStatus: 2,
expectedMessage: "could not establish public repository visibility for example/upstream — HTTP 403",
endpoints: ["api rate_limit", "api repos/example/upstream/commits/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "api repos/example/upstream"],
},
{
name: "repository metadata is unavailable",
commitStatus: 404,
metadataStatus: 500,
expectedStatus: 2,
expectedMessage: "could not establish public repository visibility for example/upstream — HTTP 500",
endpoints: ["api rate_limit", "api repos/example/upstream/commits/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "api repos/example/upstream"],
},
{
name: "repository metadata response is malformed",
commitStatus: 404,
metadataStatus: 200,
malformedMetadata: true,
expectedStatus: 2,
expectedMessage: "repository metadata for example/upstream was malformed, so repository visibility was not established",
endpoints: ["api rate_limit", "api repos/example/upstream/commits/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "api repos/example/upstream"],
},
{
name: "repository metadata omits visibility",
commitStatus: 404,
metadataStatus: 200,
omitMetadataPrivate: true,
expectedStatus: 2,
expectedMessage: "repository metadata for example/upstream was malformed, so repository visibility was not established",
endpoints: ["api rate_limit", "api repos/example/upstream/commits/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "api repos/example/upstream"],
},
{
name: "repository metadata has nonboolean visibility",
commitStatus: 404,
metadataStatus: 200,
metadataPrivate: "false",
expectedStatus: 2,
expectedMessage: "repository metadata for example/upstream was malformed, so repository visibility was not established",
endpoints: ["api rate_limit", "api repos/example/upstream/commits/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "api repos/example/upstream"],
},
])("classifies $name without mistaking repository access for object absence", (scenario) => {
// Source: https://github.com/Comfy-Org/ComfyUI_frontend/pull/16644#discussion_r3940075206
const fixture = mkdtempSync(join(tmpdir(), "pins-private-remote-"));
try {
mkdirSync(join(fixture, "docs"));
mkdirSync(join(fixture, "bin"));
const commit = "a".repeat(40);
const citedBy = ["citation-1.md", "citation-2.md", "citation-3.md", "citation-4.md"];
writeFileSync(
join(fixture, "docs", "upstream-pins.json"),
JSON.stringify({
pins: {
upstream: {
commit,
repo: "https://github.com/example/upstream",
path: "README.md",
established_by: "resolved from an immutable upstream revision with audit evidence",
sections_cited: ["1"],
cited_by: citedBy,
},
},
}),
);
for (const site of citedBy) writeFileSync(join(fixture, site), `Pinned at ${commit}.\n`);
for (let index = 0; index < 20; index += 1) writeFileSync(join(fixture, `tracked-${index}.md`), `fixture ${index}\n`);

const ghLog = join(fixture, "gh.log");
const fakeGh = join(fixture, "bin", "gh");
writeFileSync(
fakeGh,
`#!/bin/sh\nprintf '%s\\n' "$*" >> "$GH_TEST_LOG"\n` +
`if [ "$1" = "--version" ]; then echo 'gh version test'; exit 0; fi\n` +
`status=200\n` +
`case "$2" in\n` +
` rate_limit) status=200;;\n` +
` repos/example/upstream/commits/*) status="$COMMIT_STATUS";;\n` +
` repos/example/upstream/contents/*) status="$CONTENTS_STATUS";;\n` +
` repos/example/upstream) status="$METADATA_STATUS";;\n` +
` *) echo "unexpected endpoint: $2" >&2; exit 9;;\n` +
`esac\n` +
`if [ "$status" != 200 ]; then echo "gh: fixture failure (HTTP $status)" >&2; exit 1; fi\n` +
`case "$2" in\n` +
` repos/example/upstream/contents/*) printf '%s\\n' '{"content":"IyAxIFRpdGxlCg==","encoding":"base64"}';;\n` +
` repos/example/upstream) if [ "$MALFORMED_METADATA" = true ]; then echo '{}'; elif [ "$OMIT_METADATA_PRIVATE" = true ]; then echo '{"full_name":"example/upstream"}'; else printf '{"full_name":"example/upstream","private":%s}\\n' "$METADATA_PRIVATE"; fi;;\n` +
` *) echo '{}';;\n` +
`esac\n`,
);
chmodSync(fakeGh, 0o755);
expect(spawnSync("git", ["init", "--quiet"], { cwd: fixture }).status).toBe(0);
expect(spawnSync("git", ["add", "."], { cwd: fixture }).status).toBe(0);

const run = spawnSync(process.execPath, [join(root, "scripts", "check-pins.mjs"), "--verify-remote"], {
encoding: "utf8",
env: {
...process.env,
PINS_ROOT: fixture,
PATH: `${join(fixture, "bin")}:${process.env.PATH}`,
GH_TEST_LOG: ghLog,
COMMIT_STATUS: String(scenario.commitStatus),
CONTENTS_STATUS: String(scenario.contentsStatus ?? 200),
METADATA_STATUS: String(scenario.metadataStatus),
MALFORMED_METADATA: String(scenario.malformedMetadata ?? false),
OMIT_METADATA_PRIVATE: String(scenario.omitMetadataPrivate ?? false),
METADATA_PRIVATE: JSON.stringify(scenario.metadataPrivate ?? false),
},
});
expect(run.status, run.stderr).toBe(scenario.expectedStatus);
expect(run.stderr).toContain(scenario.expectedMessage);
expect(readFileSync(ghLog, "utf8").trim().split("\n")).toEqual(["--version", ...scenario.endpoints]);
} finally {
rmSync(fixture, { recursive: true, force: true });
}
});

it("makes the production gate fail on a planted moving upstream citation", () => {
const fixture = mkdtempSync(join(tmpdir(), "pins-"));
try {
Expand Down
Loading