Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion .gitleaks.toml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,21 @@ description = "Bundled TUI shortcut and permission translation key"
condition = "AND"
paths = ['''(^|/)dist/chunks/(chunk|launcher)-[A-Z0-9]+\.js$''']
regexTarget = "match"
regexes = ['''^defaultKeys: "ctrl\+shift\+down"$''', '''^labelKey: "permission\.scope\.byArgvPrefix2"$''']
regexes = ['''^defaultKeys: ?"ctrl\+shift\+down"$''', '''^labelKey: ?"permission\.scope\.byArgvPrefix2"$''']

[[rules.allowlists]]
description = "Bundled node-forge public PKCS12 algorithm names"
condition = "AND"
paths = ['''(^|/)dist/chunks/chunk-[A-Z0-9]+\.js$''']
regexTarget = "match"
regexes = ['''^pbeWithSHAAnd3-KeyTripleDES-CBC","pbewithSHAAnd40BitRC2-CBC"$''']

[[rules.allowlists]]
description = "Bundled IM binding migration guard, no credential value"
condition = "AND"
paths = ['''(^|/)dist/chunks/chunk-[A-Z0-9]+\.js$''']
regexTarget = "match"
regexes = ['''^[A-Za-z_$][A-Za-z0-9_$]*\.resolvedProjectKey\|\|[A-Za-z_$][A-Za-z0-9_$]*\.mutationReceipts===void $''']

[[rules.allowlists]]
description = "node-forge PKCS12 function alias, no key material"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import type {
ThinkingContent,
UserMessage,
} from '@earendil-works/pi-ai';
import { convertToLlm } from '@earendil-works/pi-coding-agent';
import { convertToLlm } from '@earendil-works/pi-coding-agent/messages';

import { imageDimensions } from './image-dimensions.js';

Expand Down
2 changes: 1 addition & 1 deletion packages/agent-core/src/pi-turn-runner/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {
createEditTool,
createReadTool,
createWriteTool,
} from '@earendil-works/pi-coding-agent';
} from '@earendil-works/pi-coding-agent/tools';
import { createBashEnvSpawnHook, resolveBashEnvPolicy } from '../bash-subprocess-env.js';
import type { TSchema } from '@sinclair/typebox';
import type { RuntimeTool, ToolExecutionContext } from '../tools/index.js';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* absorbed by the manager's `safetyMarginTokens` / `reserveTokens` headroom.
*/

import { convertToLlm } from '@earendil-works/pi-coding-agent';
import { convertToLlm } from '@earendil-works/pi-coding-agent/messages';
import type { AgentMessage } from '@earendil-works/pi-agent-core';
import type { Api, Model, Tool } from '@earendil-works/pi-ai';

Expand Down
4 changes: 2 additions & 2 deletions packages/agent-tools/src/desktop/local-pi-tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ import {
createEditTool,
createReadTool,
createWriteTool,
getShellConfig,
} from '@earendil-works/pi-coding-agent';
} from '@earendil-works/pi-coding-agent/tools';
import { getShellConfig } from '@earendil-works/pi-coding-agent/shell';
import type { AgentTool, AgentToolResult } from '@earendil-works/pi-agent-core';
import { access } from 'node:fs/promises';
import { isAbsolute, resolve as resolvePath } from 'node:path';
Expand Down
2 changes: 1 addition & 1 deletion packages/agent-tools/src/shared/read-guards.ts
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,7 @@ const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
/**
* Replicated from `third_party/pi-mono/packages/coding-agent/src/utils/mime.ts`
* `detectSupportedImageMimeType` — the function is NOT exported from
* `@earendil-works/pi-coding-agent` (package exposes only the root entry),
* `@earendil-works/pi-coding-agent`,
* so we keep a byte-exact copy here. KEEP IN SYNC on pi upstream syncs:
* the whole point of this replica is that the exemption face equals pi's
* image-branch acceptance face (JPEG minus JPEG-LS, PNG minus APNG, GIF,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
readJsonl,
writeJsonlAtomically,
type JsonlMalformedLine,
type JsonlReadCache,
} from './jsonl.js';
import {
decodeCanonicalHistoryArtifact,
Expand All @@ -24,6 +25,15 @@ export type {
} from './canonical-history-source.js';
export type { CanonicalHistoryArtifact } from './canonical-history-artifact.js';

// Only records decoded from file contents and recursively frozen here are trusted.
const ownedEnvelopeJson = new WeakMap<CanonicalHistoryEnvelope, string | undefined>();
const ownedRecordArrays = new WeakSet<readonly CanonicalHistoryEnvelope[]>();
const ownedRevisions = new WeakMap<readonly CanonicalHistoryEnvelope[], string>();
const ownedSequences = new WeakMap<
readonly CanonicalHistoryEnvelope[],
CanonicalHistorySequenceInspection
>();

const ENVELOPE_KEYS = new Set([
'message_id',
'turn_id',
Expand Down Expand Up @@ -268,6 +278,8 @@ export interface CanonicalHistoryEnvelope {

export interface CanonicalHistoryJsonlDataSourceOptions {
readonly activePath: string;
/** Internal readers may share frozen records; public readers retain detached values. */
readonly reuseDecodedRecords?: boolean;
readonly onMalformedLine?: (line: JsonlMalformedLine) => void;
}

Expand All @@ -294,6 +306,9 @@ export type CanonicalHistorySequenceInspection =
};

export function decodeCanonicalHistoryEnvelope(value: unknown): CanonicalHistoryEnvelope {
if (ownedEnvelopeJson.has(value as CanonicalHistoryEnvelope)) {
return value as CanonicalHistoryEnvelope;
}
const envelope = requirePlainRecord(value, 'envelope');
assertExactKeys(envelope, ENVELOPE_KEYS, ['message_id', 'turn_id', 'message'], 'envelope');
assertJsonCompatible(envelope, 'envelope');
Expand Down Expand Up @@ -358,6 +373,8 @@ export function assertCanonicalHistorySequence(records: readonly CanonicalHistor
export function inspectCanonicalHistorySequence(
records: readonly CanonicalHistoryEnvelope[],
): CanonicalHistorySequenceInspection {
const cached = ownedSequences.get(records);
if (cached) return copyInspection(cached);
const state: HistorySequenceState = {
messageIds: new Set(),
toolCallIds: new Set(),
Expand All @@ -371,14 +388,22 @@ export function inspectCanonicalHistorySequence(
validateSequenceRecord(envelope, index, state);
}

if (state.pendingToolCallIds && state.pendingToolCallIds.size > 0) {
return {
status: 'pending-tool-results',
settledPrefixLength: state.pendingToolCallStartIndex ?? records.length,
pendingToolCallIds: [...state.pendingToolCallIds],
};
}
return { status: 'settled' };
const inspection: CanonicalHistorySequenceInspection =
state.pendingToolCallIds && state.pendingToolCallIds.size > 0
? {
status: 'pending-tool-results',
settledPrefixLength: state.pendingToolCallStartIndex ?? records.length,
pendingToolCallIds: [...state.pendingToolCallIds],
}
: { status: 'settled' };
if (ownedRecordArrays.has(records)) ownedSequences.set(records, copyInspection(inspection));
return inspection;
}

function copyInspection(value: CanonicalHistorySequenceInspection): CanonicalHistorySequenceInspection {
return value.status === 'settled'
? { status: 'settled' }
: { ...value, pendingToolCallIds: [...value.pendingToolCallIds] };
}

/**
Expand All @@ -389,6 +414,8 @@ export function inspectCanonicalHistorySequence(
* not provide a cross-process writer lock.
*/
export class CanonicalHistoryJsonlDataSource {
private readonly readCache: JsonlReadCache<CanonicalHistoryEnvelope> = { bytes: Buffer.alloc(0), records: [] };

constructor(private readonly options: CanonicalHistoryJsonlDataSourceOptions) {}

async readActive(): Promise<CanonicalHistoryEnvelope[]> {
Expand All @@ -398,7 +425,7 @@ export class CanonicalHistoryJsonlDataSource {
}

async readActiveStrict(filePath = this.options.activePath): Promise<CanonicalHistoryEnvelope[]> {
const records = await readStrictEnvelopeFile(filePath);
const records = await this.readEnvelopesStrict(filePath);
inspectCanonicalHistorySequence(records);
return records;
}
Expand All @@ -407,7 +434,11 @@ export class CanonicalHistoryJsonlDataSource {
async readEnvelopesStrict(
filePath = this.options.activePath,
): Promise<CanonicalHistoryEnvelope[]> {
return readStrictEnvelopeFile(filePath);
if (!this.options.reuseDecodedRecords) return readStrictEnvelopeFile(filePath);
const records = await readJsonl(filePath, decodeOwnedEnvelope, undefined, this.readCache);
Object.freeze(records);
ownedRecordArrays.add(records);
return records;
}

async readStrict(filePath = this.options.activePath): Promise<CanonicalHistoryEnvelope[]> {
Expand Down Expand Up @@ -555,6 +586,7 @@ function normalizeActiveRecords(
}

function decodeRecords(records: readonly CanonicalHistoryEnvelope[]): CanonicalHistoryEnvelope[] {
if (ownedRecordArrays.has(records)) return records as CanonicalHistoryEnvelope[];
const decoded: CanonicalHistoryEnvelope[] = [];
for (let index = 0; index < records.length; index += 1) {
if (!Object.hasOwn(records, index)) invalidEnvelope(`records[${String(index)}] is sparse`);
Expand All @@ -564,14 +596,37 @@ function decodeRecords(records: readonly CanonicalHistoryEnvelope[]): CanonicalH
}

function revisionOfNormalized(records: readonly CanonicalHistoryEnvelope[]): string {
const cached = ownedRevisions.get(records);
if (cached !== undefined) return cached;
// Preserve the canonical JSON array bytes without building a sorted copy and
// serialized string of the entire history at once.
const hash = createHash('sha256').update('[');
for (let index = 0; index < records.length; index += 1) {
if (index > 0) hash.update(',');
hash.update(canonicalJson(records[index]), 'utf8');
const record = records[index]!;
let serialized = ownedEnvelopeJson.get(record);
if (serialized === undefined) {
serialized = canonicalJson(record);
if (ownedEnvelopeJson.has(record)) ownedEnvelopeJson.set(record, serialized);
}
hash.update(serialized, 'utf8');
}
return `sha256:${hash.update(']').digest('hex')}`;
const revision = `sha256:${hash.update(']').digest('hex')}`;
if (ownedRecordArrays.has(records)) ownedRevisions.set(records, revision);
return revision;
}

function decodeOwnedEnvelope(value: unknown): CanonicalHistoryEnvelope {
const record = decodeCanonicalHistoryEnvelope(value);
freezeDecodedJson(record);
ownedEnvelopeJson.set(record, undefined);
return record;
}

function freezeDecodedJson(value: unknown): void {
if (value === null || typeof value !== 'object' || Object.isFrozen(value)) return;
for (const child of Object.values(value)) freezeDecodedJson(child);
Object.freeze(value);
}

function decodeMessage(value: unknown): CanonicalHistoryMessage {
Expand Down
67 changes: 59 additions & 8 deletions packages/local-runtime-v2/src/infra/file/jsonl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,34 +32,85 @@ export class JsonlAppendCommitUncertainError extends Error {
}
}

export interface JsonlReadCache<T> {
bytes: Buffer;
records: readonly T[];
}

export async function readJsonl<T>(
filePath: string,
decode: (value: unknown) => T,
onMalformedLine?: (line: JsonlMalformedLine) => void,
/** Strict private reads return immutable arrays when a cache is supplied. */
readCache?: JsonlReadCache<T>,
): Promise<T[]> {
// Tolerant readers must still report every malformed line on every read.
if (readCache && !onMalformedLine) return readCachedJsonl(filePath, decode, readCache);
const contents = await readFile(filePath, 'utf-8');
if (contents.length === 0) return [];

const records: T[] = [];
const lines = contents.split('\n');
if (lines.at(-1) === '') lines.pop();
const records: T[] = [];
for (const [index, line] of lines.entries()) {
try {
if (line.trim().length === 0) throw new Error('blank line');
records.push(decode(parseJsonLine(line)));
} catch (error) {
const malformed = {
path: filePath,
lineNo: index + 1,
reason: errorReason(error),
};
const malformed = { path: filePath, lineNo: index + 1, reason: errorReason(error) };
if (!onMalformedLine) throw malformedLineError(malformed);
onMalformedLine(malformed);
}
}
return records;
}

async function readCachedJsonl<T>(
filePath: string,
decode: (value: unknown) => T,
cache: JsonlReadCache<T>,
): Promise<T[]> {
// Always read fresh bytes: timestamps and file size cannot prove an unchanged
// prefix. Compare before decoding to avoid allocating a whole-history string.
const bytes = await readFile(filePath);
if (bytes.equals(cache.bytes)) return cache.records as T[];
const reuse = cache.bytes.at(-1) === 10 &&
bytes.length >= cache.bytes.length &&
bytes.subarray(0, cache.bytes.length).equals(cache.bytes);
const records: T[] = reuse ? [...cache.records] : [];
let offset = reuse ? cache.bytes.length : 0;
const limit = 4 * 1024 * 1024;
let retainedEnd = offset;
let retainedRecords = records.length;
while (offset < bytes.length) {
const newline = bytes.indexOf(10, offset);
const end = newline === -1 ? bytes.length : newline;
try {
// Decode each line separately so parsed values cannot retain a string
// slice of an older whole file.
const line = bytes.toString('utf8', offset, end);
if (line.trim().length === 0) throw new Error('blank line');
records.push(decode(parseJsonLine(line)));
} catch (error) {
throw malformedLineError({
path: filePath, lineNo: records.length + 1, reason: errorReason(error),
});
}
offset = newline === -1 ? bytes.length : newline + 1;
if (newline !== -1 && offset <= limit) {
retainedEnd = offset;
retainedRecords = records.length;
}
}
if (bytes.length <= limit) {
cache.bytes = bytes;
cache.records = Object.freeze(records);
} else {
// Copy the bounded prefix so it cannot retain the entire file's buffer.
cache.bytes = Buffer.from(bytes.subarray(0, retainedEnd));
cache.records = Object.freeze(records.slice(0, retainedRecords));
}
return Object.freeze(records) as T[];
}

function parseJsonLine(line: string): unknown {
try {
return JSON.parse(line) as unknown;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { StringDecoder } from 'node:string_decoder';

import { createBashTool, type BashOperations } from '@earendil-works/pi-coding-agent';
import { createBashTool, type BashOperations } from '@earendil-works/pi-coding-agent/tools';
import {
createBashEnvSpawnHook,
type BashEnvPolicy,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { join } from "node:path";
import { AuthStorage } from "@earendil-works/pi-coding-agent";
import { AuthStorage } from '@earendil-works/pi-coding-agent/auth-storage';
import { afterEach, describe, expect, it, vi } from "vitest";

import type {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { randomUUID } from 'node:crypto';
import { join } from 'node:path';

import { AuthStorage } from '@earendil-works/pi-coding-agent';
import { AuthStorage } from '@earendil-works/pi-coding-agent/auth-storage';

import type {
LocalByokConfigDraft,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { createLocalBashOperations, type BashOperations } from '@earendil-works/pi-coding-agent';
import { createLocalBashOperations, type BashOperations } from '@earendil-works/pi-coding-agent/tools';
import type { BashEnvPolicy } from '@mavis/agent-core/bash-subprocess-env';
import type { LocalSandboxBashOperationsFactory } from '@mavis/agent-tools/desktop';

Expand Down
Loading
Loading