Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 43 additions & 4 deletions bin/knowledge-mcp.js
Original file line number Diff line number Diff line change
Expand Up @@ -23725,14 +23725,49 @@ REDACTION_PATTERNS.push(...COMMON_BARE_TOKEN_PATTERNS);

// src/private-ref.ts
import { createHash as createHash3 } from "crypto";
import { realpathSync } from "fs";
import { homedir as homedir3, tmpdir } from "os";
var PATH_TAIL = String.raw`[^\s"'<>),\]}]`;
var PATH_SEGMENT = String.raw`[^/\\\s"'<>]+`;
var FILE_URI_RE = /file:\/\/[^\s"'<>),\]}]+/gi;
var ABSOLUTE_HASNA_PATH_RE = /\/[^\s"'<>),\]}]*\.hasna\/[^\s"'<>),\]}]*/g;
var ABSOLUTE_LOCAL_PATH_RE = /\/(?:home|tmp)\/[^\s"'<>),\]}]+/g;
var HASNA_PATH_RE = /(?:~|\/home\/[^/\s"'<>]+)?\/?\.hasna(?:\/[^\s"'<>),\]}]*)?/gi;
var PRIVATE_WORKSPACE_PATH_RE = /\/home\/[^/\s"'<>]+\/(?:workspace|Workspace)\/[^\s"'<>),\]}]*/g;
var HASNA_PATH_RE = /(?:~|\/(?:home|Users)\/[^/\s"'<>]+)?\/?\.hasna(?:\/[^\s"'<>),\]}]*)?/gi;
var PRIVATE_WORKSPACE_PATH_RE = new RegExp(String.raw`/(?:home|Users)/${PATH_SEGMENT}/(?:workspace|Workspace)/${PATH_TAIL}*`, "g");
var LOCAL_PATH_PATTERNS = [
new RegExp(String.raw`/(?:home|Users)/${PATH_SEGMENT}(?:/${PATH_TAIL}*)?`, "g"),
new RegExp(String.raw`(?:/private)?/var/(?:folders|tmp)/${PATH_TAIL}+`, "g"),
new RegExp(String.raw`(?:/private)?/tmp/${PATH_TAIL}+`, "g"),
new RegExp(String.raw`(?<![A-Za-z0-9])[A-Za-z]:[\\/]${PATH_TAIL}+`, "g"),
new RegExp(String.raw`\\\\${PATH_SEGMENT}\\${PATH_TAIL}+`, "g")
];
var RAW_DB_OR_ENV_RE = /\b(?:knowledge\.db(?:[-.][A-Za-z0-9_-]+)?|db\.json(?:[-.][A-Za-z0-9_-]+)?|cloud\.env|migration-exports\/[^\s"'<>),\]}]+)\b/gi;
var DATABASE_URL_RE = /\b(?:postgres(?:ql)?|mysql|mariadb):\/\/[^\s"'<>),\]}]+/gi;
var OPAQUE_EXPORT_KEYS = new Set(["content_base64"]);
function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
function isUsableRoot(root) {
return root.length >= 4 && root !== "/" && !/^[A-Za-z]:[\\/]?$/.test(root);
}
var hostRootCacheKey = null;
var hostRootCache = [];
function hostRootPatterns() {
const roots = new Set;
for (const root of [homedir3(), tmpdir()]) {
if (!root)
continue;
roots.add(root);
try {
roots.add(realpathSync(root));
} catch {}
}
const key = [...roots].sort().join("\x00");
if (key === hostRootCacheKey)
return hostRootCache;
hostRootCacheKey = key;
hostRootCache = [...roots].filter(isUsableRoot).sort((a, b) => b.length - a.length).map((root) => new RegExp(`${escapeRegExp(root)}(?:[/\\\\]${PATH_TAIL}*)?`, "g"));
return hostRootCache;
}
function fingerprint(value) {
return createHash3("sha256").update(value).digest("hex").slice(0, 12);
}
Expand Down Expand Up @@ -23820,7 +23855,11 @@ function assertNoPrivateRefs(value, options = {}) {
throw new Error(`Knowledge private-ref lint failed (${summary}). Store open-files/s3 refs or approved runtime secret refs instead of private .hasna, file://, raw DB/export, or cloud.env refs.`);
}
function redactString(value) {
return redactSecrets(value).text.replace(DATABASE_URL_RE, (match) => `[REDACTED:database-url:${fingerprint(match)}]`).replace(FILE_URI_RE, (match) => `[REDACTED:local-file-uri:${fingerprint(match)}]`).replace(ABSOLUTE_HASNA_PATH_RE, (match) => `[REDACTED:local-hasna-path:${fingerprint(match)}]`).replace(PRIVATE_WORKSPACE_PATH_RE, (match) => `[REDACTED:private-workspace:${fingerprint(match)}]`).replace(ABSOLUTE_LOCAL_PATH_RE, (match) => `[REDACTED:local-path:${fingerprint(match)}]`).replace(HASNA_PATH_RE, (match) => `[REDACTED:hasna-path:${fingerprint(match)}]`).replace(RAW_DB_OR_ENV_RE, (match) => `[REDACTED:private-artifact:${fingerprint(match)}]`);
let text = redactSecrets(value).text.replace(DATABASE_URL_RE, (match) => `[REDACTED:database-url:${fingerprint(match)}]`).replace(FILE_URI_RE, (match) => `[REDACTED:local-file-uri:${fingerprint(match)}]`).replace(ABSOLUTE_HASNA_PATH_RE, (match) => `[REDACTED:local-hasna-path:${fingerprint(match)}]`).replace(PRIVATE_WORKSPACE_PATH_RE, (match) => `[REDACTED:private-workspace:${fingerprint(match)}]`);
for (const pattern of [...hostRootPatterns(), ...LOCAL_PATH_PATTERNS]) {
text = text.replace(pattern, (match) => `[REDACTED:local-path:${fingerprint(match)}]`);
}
return text.replace(HASNA_PATH_RE, (match) => `[REDACTED:hasna-path:${fingerprint(match)}]`).replace(RAW_DB_OR_ENV_RE, (match) => `[REDACTED:private-artifact:${fingerprint(match)}]`);
}
function redactPrivateRefs(value) {
if (typeof value === "string")
Expand Down
394 changes: 197 additions & 197 deletions bin/knowledge.js

Large diffs are not rendered by default.

47 changes: 43 additions & 4 deletions dist/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -25019,14 +25019,49 @@ REDACTION_PATTERNS.push(...COMMON_BARE_TOKEN_PATTERNS);

// src/private-ref.ts
import { createHash as createHash5 } from "crypto";
import { realpathSync } from "fs";
import { homedir as homedir3, tmpdir } from "os";
var PATH_TAIL = String.raw`[^\s"'<>),\]}]`;
var PATH_SEGMENT = String.raw`[^/\\\s"'<>]+`;
var FILE_URI_RE = /file:\/\/[^\s"'<>),\]}]+/gi;
var ABSOLUTE_HASNA_PATH_RE = /\/[^\s"'<>),\]}]*\.hasna\/[^\s"'<>),\]}]*/g;
var ABSOLUTE_LOCAL_PATH_RE = /\/(?:home|tmp)\/[^\s"'<>),\]}]+/g;
var HASNA_PATH_RE = /(?:~|\/home\/[^/\s"'<>]+)?\/?\.hasna(?:\/[^\s"'<>),\]}]*)?/gi;
var PRIVATE_WORKSPACE_PATH_RE = /\/home\/[^/\s"'<>]+\/(?:workspace|Workspace)\/[^\s"'<>),\]}]*/g;
var HASNA_PATH_RE = /(?:~|\/(?:home|Users)\/[^/\s"'<>]+)?\/?\.hasna(?:\/[^\s"'<>),\]}]*)?/gi;
var PRIVATE_WORKSPACE_PATH_RE = new RegExp(String.raw`/(?:home|Users)/${PATH_SEGMENT}/(?:workspace|Workspace)/${PATH_TAIL}*`, "g");
var LOCAL_PATH_PATTERNS = [
new RegExp(String.raw`/(?:home|Users)/${PATH_SEGMENT}(?:/${PATH_TAIL}*)?`, "g"),
new RegExp(String.raw`(?:/private)?/var/(?:folders|tmp)/${PATH_TAIL}+`, "g"),
new RegExp(String.raw`(?:/private)?/tmp/${PATH_TAIL}+`, "g"),
new RegExp(String.raw`(?<![A-Za-z0-9])[A-Za-z]:[\\/]${PATH_TAIL}+`, "g"),
new RegExp(String.raw`\\\\${PATH_SEGMENT}\\${PATH_TAIL}+`, "g")
];
var RAW_DB_OR_ENV_RE = /\b(?:knowledge\.db(?:[-.][A-Za-z0-9_-]+)?|db\.json(?:[-.][A-Za-z0-9_-]+)?|cloud\.env|migration-exports\/[^\s"'<>),\]}]+)\b/gi;
var DATABASE_URL_RE = /\b(?:postgres(?:ql)?|mysql|mariadb):\/\/[^\s"'<>),\]}]+/gi;
var OPAQUE_EXPORT_KEYS = new Set(["content_base64"]);
function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
function isUsableRoot(root) {
return root.length >= 4 && root !== "/" && !/^[A-Za-z]:[\\/]?$/.test(root);
}
var hostRootCacheKey = null;
var hostRootCache = [];
function hostRootPatterns() {
const roots = new Set;
for (const root of [homedir3(), tmpdir()]) {
if (!root)
continue;
roots.add(root);
try {
roots.add(realpathSync(root));
} catch {}
}
const key = [...roots].sort().join("\x00");
if (key === hostRootCacheKey)
return hostRootCache;
hostRootCacheKey = key;
hostRootCache = [...roots].filter(isUsableRoot).sort((a, b) => b.length - a.length).map((root) => new RegExp(`${escapeRegExp(root)}(?:[/\\\\]${PATH_TAIL}*)?`, "g"));
return hostRootCache;
}
function fingerprint(value) {
return createHash5("sha256").update(value).digest("hex").slice(0, 12);
}
Expand Down Expand Up @@ -25114,7 +25149,11 @@ function assertNoPrivateRefs(value, options = {}) {
throw new Error(`Knowledge private-ref lint failed (${summary}). Store open-files/s3 refs or approved runtime secret refs instead of private .hasna, file://, raw DB/export, or cloud.env refs.`);
}
function redactString(value) {
return redactSecrets(value).text.replace(DATABASE_URL_RE, (match) => `[REDACTED:database-url:${fingerprint(match)}]`).replace(FILE_URI_RE, (match) => `[REDACTED:local-file-uri:${fingerprint(match)}]`).replace(ABSOLUTE_HASNA_PATH_RE, (match) => `[REDACTED:local-hasna-path:${fingerprint(match)}]`).replace(PRIVATE_WORKSPACE_PATH_RE, (match) => `[REDACTED:private-workspace:${fingerprint(match)}]`).replace(ABSOLUTE_LOCAL_PATH_RE, (match) => `[REDACTED:local-path:${fingerprint(match)}]`).replace(HASNA_PATH_RE, (match) => `[REDACTED:hasna-path:${fingerprint(match)}]`).replace(RAW_DB_OR_ENV_RE, (match) => `[REDACTED:private-artifact:${fingerprint(match)}]`);
let text = redactSecrets(value).text.replace(DATABASE_URL_RE, (match) => `[REDACTED:database-url:${fingerprint(match)}]`).replace(FILE_URI_RE, (match) => `[REDACTED:local-file-uri:${fingerprint(match)}]`).replace(ABSOLUTE_HASNA_PATH_RE, (match) => `[REDACTED:local-hasna-path:${fingerprint(match)}]`).replace(PRIVATE_WORKSPACE_PATH_RE, (match) => `[REDACTED:private-workspace:${fingerprint(match)}]`);
for (const pattern of [...hostRootPatterns(), ...LOCAL_PATH_PATTERNS]) {
text = text.replace(pattern, (match) => `[REDACTED:local-path:${fingerprint(match)}]`);
}
return text.replace(HASNA_PATH_RE, (match) => `[REDACTED:hasna-path:${fingerprint(match)}]`).replace(RAW_DB_OR_ENV_RE, (match) => `[REDACTED:private-artifact:${fingerprint(match)}]`);
}
function redactPrivateRefs(value) {
if (typeof value === "string")
Expand Down
97 changes: 91 additions & 6 deletions src/private-ref.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import { createHash } from 'node:crypto';
import { realpathSync } from 'node:fs';
import { homedir, tmpdir } from 'node:os';
import { redactSecrets } from './safety';

export type PrivateRefIssueSeverity = 'medium' | 'high';
Expand All @@ -15,15 +17,93 @@ export interface PrivateRefLintOptions {
allowPrivateWorkspaceRefs?: boolean;
}

/** Characters that terminate a filesystem reference inside serialized JSON, log
* lines, and Markdown. Shared by every path pattern so redaction boundaries stay
* consistent across OS families. */
const PATH_TAIL = String.raw`[^\s"'<>),\]}]`;
/** A single path segment: a user name, host name, or directory. */
const PATH_SEGMENT = String.raw`[^/\\\s"'<>]+`;

const FILE_URI_RE = /file:\/\/[^\s"'<>),\]}]+/gi;
const ABSOLUTE_HASNA_PATH_RE = /\/[^\s"'<>),\]}]*\.hasna\/[^\s"'<>),\]}]*/g;
const ABSOLUTE_LOCAL_PATH_RE = /\/(?:home|tmp)\/[^\s"'<>),\]}]+/g;
const HASNA_PATH_RE = /(?:~|\/home\/[^/\s"'<>]+)?\/?\.hasna(?:\/[^\s"'<>),\]}]*)?/gi;
const PRIVATE_WORKSPACE_PATH_RE = /\/home\/[^/\s"'<>]+\/(?:workspace|Workspace)\/[^\s"'<>),\]}]*/g;
const HASNA_PATH_RE = /(?:~|\/(?:home|Users)\/[^/\s"'<>]+)?\/?\.hasna(?:\/[^\s"'<>),\]}]*)?/gi;
const PRIVATE_WORKSPACE_PATH_RE = new RegExp(
String.raw`/(?:home|Users)/${PATH_SEGMENT}/(?:workspace|Workspace)/${PATH_TAIL}*`,
'g',
);

/**
* Absolute local filesystem shapes, independent of the OS that produced them.
*
* These MUST NOT be narrowed to the exporting host's own OS. A bundle exported on
* macOS or Windows has to redact as thoroughly as one exported on Linux, and rows
* pulled from a peer can carry paths from a different OS than the machine doing the
* export. Matching only /home and /tmp is what let macOS temp paths
* (/var/folders/...) and macOS home paths (/Users/...) reach exported bundles
* intact — a leak that reproduced solely on the macOS CI runners.
*/
const LOCAL_PATH_PATTERNS: readonly RegExp[] = [
// POSIX home directories: Linux /home/<user>, macOS /Users/<user>.
new RegExp(String.raw`/(?:home|Users)/${PATH_SEGMENT}(?:/${PATH_TAIL}*)?`, 'g'),
// macOS per-user temp dirs, both as os.tmpdir() reports them (/var/folders/...)
// and as realpath resolves them (/private/var/folders/...).
new RegExp(String.raw`(?:/private)?/var/(?:folders|tmp)/${PATH_TAIL}+`, 'g'),
// POSIX temp, plus the macOS /private realpath spelling.
new RegExp(String.raw`(?:/private)?/tmp/${PATH_TAIL}+`, 'g'),
// Windows drive-absolute paths (C:\... and C:/...). The lookbehind keeps URL
// schemes such as https:// and s3:// from being read as a single-letter drive.
new RegExp(String.raw`(?<![A-Za-z0-9])[A-Za-z]:[\\/]${PATH_TAIL}+`, 'g'),
// Windows UNC shares (\\host\share\...).
new RegExp(String.raw`\\\\${PATH_SEGMENT}\\${PATH_TAIL}+`, 'g'),
];
const RAW_DB_OR_ENV_RE = /\b(?:knowledge\.db(?:[-.][A-Za-z0-9_-]+)?|db\.json(?:[-.][A-Za-z0-9_-]+)?|cloud\.env|migration-exports\/[^\s"'<>),\]}]+)\b/gi;
const DATABASE_URL_RE = /\b(?:postgres(?:ql)?|mysql|mariadb):\/\/[^\s"'<>),\]}]+/gi;
const OPAQUE_EXPORT_KEYS = new Set(['content_base64']);

function escapeRegExp(value: string): string {
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}

/** A root broad enough to swallow unrelated text is worse than no pattern at all,
* so degenerate roots ("/", "C:\") are refused rather than turned into patterns. */
function isUsableRoot(root: string): boolean {
return root.length >= 4 && root !== '/' && !/^[A-Za-z]:[\\/]?$/.test(root);
}

let hostRootCacheKey: string | null = null;
let hostRootCache: readonly RegExp[] = [];

/**
* Patterns for the running host's own home and temp roots.
*
* The OS is the source of truth for where this machine keeps private files, so the
* roots are read from it rather than hardcoded. This is what covers hosts whose
* roots match none of the conventional shapes above — a container with
* HOME=/github/home, or TMPDIR pointed at a scratch volume. Both the reported and
* the realpath-resolved spelling are covered because macOS reports tmpdir as
* /var/folders/... while resolving it to /private/var/folders/...
*/
function hostRootPatterns(): readonly RegExp[] {
const roots = new Set<string>();
for (const root of [homedir(), tmpdir()]) {
if (!root) continue;
roots.add(root);
try {
roots.add(realpathSync(root));
} catch {
// An unresolvable root still gets its reported spelling covered above.
}
}
const key = [...roots].sort().join('\u0000');
if (key === hostRootCacheKey) return hostRootCache;
hostRootCacheKey = key;
hostRootCache = [...roots]
.filter(isUsableRoot)
.sort((a, b) => b.length - a.length)
.map((root) => new RegExp(`${escapeRegExp(root)}(?:[/\\\\]${PATH_TAIL}*)?`, 'g'));
return hostRootCache;
}

function fingerprint(value: string): string {
return createHash('sha256').update(value).digest('hex').slice(0, 12);
}
Expand Down Expand Up @@ -122,12 +202,17 @@ export function assertNoPrivateRefs(value: unknown, options: PrivateRefLintOptio
}

function redactString(value: string): string {
return redactSecrets(value).text
// Most specific categories first, so a workspace or .hasna path keeps its own
// label instead of being flattened into the generic local-path category.
let text = redactSecrets(value).text
.replace(DATABASE_URL_RE, (match) => `[REDACTED:database-url:${fingerprint(match)}]`)
.replace(FILE_URI_RE, (match) => `[REDACTED:local-file-uri:${fingerprint(match)}]`)
.replace(ABSOLUTE_HASNA_PATH_RE, (match) => `[REDACTED:local-hasna-path:${fingerprint(match)}]`)
.replace(PRIVATE_WORKSPACE_PATH_RE, (match) => `[REDACTED:private-workspace:${fingerprint(match)}]`)
.replace(ABSOLUTE_LOCAL_PATH_RE, (match) => `[REDACTED:local-path:${fingerprint(match)}]`)
.replace(PRIVATE_WORKSPACE_PATH_RE, (match) => `[REDACTED:private-workspace:${fingerprint(match)}]`);
for (const pattern of [...hostRootPatterns(), ...LOCAL_PATH_PATTERNS]) {
text = text.replace(pattern, (match) => `[REDACTED:local-path:${fingerprint(match)}]`);
}
return text
.replace(HASNA_PATH_RE, (match) => `[REDACTED:hasna-path:${fingerprint(match)}]`)
.replace(RAW_DB_OR_ENV_RE, (match) => `[REDACTED:private-artifact:${fingerprint(match)}]`);
}
Expand Down
97 changes: 97 additions & 0 deletions tests/private-ref.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
import { describe, expect, test } from 'bun:test';
import { homedir, tmpdir } from 'node:os';
import { join } from 'node:path';
import { redactPrivateRefs } from '../src/private-ref';

/**
* Local filesystem path shapes, one per OS family, planted verbatim so that these
* assertions hold whichever OS is running the suite. The values are synthetic
* fixtures — they must never be replaced with a real host's paths, and failures
* report the CATEGORY LABEL rather than the offending value, because a guard that
* prints what it caught publishes it.
*
* Before platform-agnostic redaction, only the /home/... and /tmp/... forms were
* matched. Every other shape survived export untouched, which is why the leak
* reproduced on the macOS CI runners (temp dirs live under /var/folders) and never
* on Linux.
*/
const PLANTED_LOCAL_PATHS: Record<string, string> = {
linux_home: '/home/fixture-user/workspace/private-notes.md',
linux_temp: '/tmp/fixture-export-staging/private-notes.md',
macos_home: '/Users/fixture-user/workspace/private-notes.md',
macos_temp: '/var/folders/1a/fixturehash0000gn/T/fixture-export/private-notes.md',
macos_private_temp: '/private/var/folders/1a/fixturehash0000gn/T/fixture-export/private-notes.md',
macos_var_temp: '/var/tmp/fixture-export-staging/private-notes.md',
windows_drive_backslash: 'C:\\Users\\fixture-user\\AppData\\Local\\Temp\\fixture-export\\private-notes.md',
windows_drive_forwardslash: 'D:/fixture-user/workspace/private-notes.md',
windows_unc: '\\\\fixture-host\\share\\fixture-export\\private-notes.md',
};

/** Category labels whose planted value survived redaction. Never returns values. */
function survivingCategories(redacted: string): string[] {
return Object.entries(PLANTED_LOCAL_PATHS)
.filter(([, value]) => redacted.includes(value))
.map(([label]) => label)
.sort();
}

describe('private ref redaction is platform-agnostic', () => {
test('redacts local filesystem paths from every OS family, not just Linux roots', () => {
const planted = Object.entries(PLANTED_LOCAL_PATHS)
.map(([label, value]) => `${label} cites ${value}`)
.join('\n');

const redacted = redactPrivateRefs(planted);

expect(survivingCategories(redacted)).toEqual([]);
expect(redacted).toContain('[REDACTED:');
});

test('redacts local paths nested in objects, arrays, and JSON-in-string columns', () => {
// Mirrors how the sync exporter actually carries paths: metadata_json is a
// JSON document stored as a TEXT column, so the path is nested inside a string.
const row = {
id: 'src_fixture',
metadata_json: JSON.stringify({
metadata: { path: PLANTED_LOCAL_PATHS.macos_temp },
path: PLANTED_LOCAL_PATHS.windows_drive_backslash,
}),
nested: [{ deep: { path: PLANTED_LOCAL_PATHS.macos_home } }],
};

const redacted = redactPrivateRefs(row);

expect(survivingCategories(JSON.stringify(redacted))).toEqual([]);
expect(redacted.id).toBe('src_fixture');
});

test("redacts this host's own home and temp roots whatever they happen to be", () => {
// The running host's roots are the authoritative answer to "is this local",
// so redaction must read them from the OS instead of assuming Linux defaults.
const hostPaths = [join(homedir(), 'fixture-export', 'private-notes.md'), join(tmpdir(), 'fixture-export', 'private-notes.md')];

for (const hostPath of hostPaths) {
const redacted = redactPrivateRefs(`generated artifact cites ${hostPath}`);
expect(redacted).not.toContain(hostPath);
expect(redacted).toContain('[REDACTED:');
}
});

test('leaves portable, non-filesystem references intact', () => {
// Over-redaction is its own defect: relative artifact keys, content hashes,
// media types, and remote URIs must survive so bundles stay importable.
const portable = [
'wiki/README.md',
'logs/2026/07/27.jsonl',
'sha256:e3ae2867534c78201b653940ad2f55bd0ff67b27f25f0c8181fc34d555717c69',
'text/markdown',
'application/x-ndjson',
's3://fixture-bucket/org/project/knowledge/wiki/README.md',
'open-files://source/src_fixture/path/wiki/README.md',
];

for (const value of portable) {
expect(redactPrivateRefs(value)).toBe(value);
}
});
});
Loading