From aef3790a91c10d9c08b396bb304bfbbb10208a13 Mon Sep 17 00:00:00 2001 From: Hugo Richard Date: Mon, 31 Aug 2026 09:44:28 +0100 Subject: [PATCH] feat(sdk): return paged list results so models stop repeating the same page REST list tools now expose hasMore and nextPage, restore page on schemas that dropped it, and cap tree/diff payloads in toModelOutput. --- .changeset/paged-list-has-more.md | 5 ++ .github/CONTRIBUTING.md | 2 +- AGENTS.md | 2 + .../content/docs/4.guide/6.working-context.md | 9 ++- .../content/docs/5.api/1.tools-catalog.md | 8 +- apps/docs/content/docs/5.api/2.reference.md | 2 +- apps/docs/skills/github-tools-agents/SKILL.md | 2 +- packages/github-tools/README.md | 10 +-- packages/github-tools/src/agents.ts | 3 +- packages/github-tools/src/core/bundles.ts | 16 ++-- packages/github-tools/src/core/checks.ts | 16 ++-- packages/github-tools/src/core/commits.ts | 17 ++-- packages/github-tools/src/core/gists.ts | 17 ++-- packages/github-tools/src/core/issues.ts | 33 ++++---- .../src/core/model-output.test.ts | 76 ++++++++++++++++++ .../github-tools/src/core/model-output.ts | 80 +++++++++++++++++-- .../github-tools/src/core/notifications.ts | 9 ++- .../github-tools/src/core/pagination.test.ts | 69 ++++++++++++++++ packages/github-tools/src/core/pagination.ts | 55 +++++++++++-- .../github-tools/src/core/pull-requests.ts | 33 ++++---- packages/github-tools/src/core/reactions.ts | 16 ++-- packages/github-tools/src/core/releases.ts | 17 ++-- packages/github-tools/src/core/repository.ts | 37 ++++++--- packages/github-tools/src/core/workflows.ts | 21 ++--- packages/github-tools/src/eve/build.test.ts | 1 + packages/github-tools/src/eve/registry.ts | 2 + packages/github-tools/src/tools/repository.ts | 3 +- 27 files changed, 423 insertions(+), 138 deletions(-) create mode 100644 .changeset/paged-list-has-more.md create mode 100644 packages/github-tools/src/core/model-output.test.ts create mode 100644 packages/github-tools/src/core/pagination.test.ts diff --git a/.changeset/paged-list-has-more.md b/.changeset/paged-list-has-more.md new file mode 100644 index 0000000..5a72d6a --- /dev/null +++ b/.changeset/paged-list-has-more.md @@ -0,0 +1,5 @@ +--- +'@github-tools/sdk': minor +--- + +REST list tools now return `{ items, hasMore, page, perPage, nextPage }` instead of a bare array. Object-shaped lists (`listCheckRuns`, `listWorkflowRuns`, `listWorkflows`, `listWorkflowJobs`, reactions) add the same paging fields next to their existing keys. When `hasMore` is true, call again with `nextPage` (or raise `maxPages`) — do not repeat the same page. `page` is restored on `listCommits`, `listIssues`, `listPullRequests`, `listCheckRuns`, `listReleases`, and `listBranches`. Filter `listCommits` with `path` / `author` / `since` / `until`. `getRepositoryTree` accepts a `path` prefix; tree and large diffs are capped in the model-facing output. diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 676b97b..c66db34 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -47,7 +47,7 @@ pnpm --filter @github-tools/sdk typecheck # Type-check the SDK Every tool splits into a **core** function (pure logic) and a **tool factory** (the `ai` SDK wrapper). See `getGistCore`/`getGist` (`packages/github-tools/src/core/gists.ts` / `src/tools/gists.ts`) for a read tool, `createIssue` (`src/tools/issues.ts`) for a write tool. -1. **Core logic** — add `{name}InputSchema` (zod, `.describe()` on every field), `{name}Description`, and `{name}Core({ token, ...args })` to `packages/github-tools/src/core/{domain}.ts`. Shape the return — never return the raw Octokit response. +1. **Core logic** — add `{name}InputSchema` (zod, `.describe()` on every field), `{name}Description`, and `{name}Core({ token, ...args })` to `packages/github-tools/src/core/{domain}.ts`. Shape the return — never return the raw Octokit response. REST list tools return `pagedList(...)` (`{ items, hasMore, page, perPage, nextPage? }`); object-shaped lists add the same paging fields next to existing keys (`checkRuns`, `runs`, …). 2. **Tool factory** — add the `"use step"` wrapper and the exported factory to `packages/github-tools/src/tools/{domain}.ts`. Read tools take `(token)`; write tools also take `({ needsApproval = true }: ToolOptions = {})`. 3. **Register** (new domain? add a re-export in `packages/github-tools/src/core/index.ts` too): - `packages/github-tools/src/core/catalog.ts` — add one `GITHUB_TOOL_CATALOG` entry (JSDoc, `description`, `inputSchema`, `get core()`, `write: true` for write tools, `connectScopes`). `GITHUB_TOOL_NAMES`, `GITHUB_WRITE_TOOLS`, `TOOL_CONNECT_SCOPES`, and the eve registry are all derived from it — no separate registration diff --git a/AGENTS.md b/AGENTS.md index f5d4296..b95d743 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,6 +68,8 @@ export const myTool = (token: GithubTokenInput, { needsApproval = true }: ToolOp }) ``` +**REST list tools** return `{ items, hasMore, page, perPage, nextPage? }` via `pagedList()` (`src/core/pagination.ts`). Object-shaped lists (`listCheckRuns`, `listWorkflowRuns`, …) add the same paging fields next to their existing keys. `hasMore` must live on the execute result — eve `toModelOutput` cannot see input. Restore `page` on every REST list schema. + **Adding a new tool?** Follow the checklist in [`.github/CONTRIBUTING.md`](.github/CONTRIBUTING.md#adding-a-new-tool) — registration files, chat metadata, docs, changeset. ### Key source files diff --git a/apps/docs/content/docs/4.guide/6.working-context.md b/apps/docs/content/docs/4.guide/6.working-context.md index cd32c65..8464ef2 100644 --- a/apps/docs/content/docs/4.guide/6.working-context.md +++ b/apps/docs/content/docs/4.guide/6.working-context.md @@ -51,8 +51,8 @@ Prefer one composite call over chaining several reads: | Tool | Returns | |---|---| -| `getPullRequestContext` | PR details + files + reviews (+ optional CI checks) | -| `getIssueContext` | Issue + `labelNames` + recent comments | +| `getPullRequestContext` | PR details + files + reviews (+ optional CI checks). `filesHasMore` / `reviewsHasMore` when those lists continue | +| `getIssueContext` | Issue + `labelNames` + recent comments. `commentsHasMore` when more comments exist | | `getReleaseContext` | Release + previous release + tag comparison | | `getCiFailureContext` | Combined status, failing checks, failed workflow jobs/steps | @@ -66,12 +66,13 @@ Call independent follow-up reads **in the same step** when you already know the | `getIssueContext` | Defaults to `detail: 'full'` (one-shot) and returns `labelNames` (strings), not full label objects | `detail: 'summary'`; use `listLabels` for descriptions; use `listIssueComments` to paginate beyond the embedded comments | | `includePatch: false` | Omits diff patches on `listPullRequestFiles`, `getCommit`, `compareCommits` | `includePatch: true`; optionally `filenames` on `listPullRequestFiles` | | File ranges | Prefer `startLine` / `endLine` / `maxLines` on `getFileContent` | Omit ranges only for small files | -| `maxPages` | List tools fetch one page by default | Set `maxPages` to combine sequential pages in one call | +| Paged lists | REST list tools return `{ items, hasMore, page, nextPage }` (or add those fields next to `checkRuns` / `runs` / …). Default is one page. | When `hasMore`, call with `nextPage` or set `maxPages` to combine pages. Do not repeat the same page. Filter `listCommits` with `path` / `author` / `since` / `until` | +| `getRepositoryTree` | Optional `path` prefix; model output caps at 200 entries | Prefer `path` over `recursive: true`; if `truncated`, narrow `path` | | Text-match fragments | `searchCode` truncates each snippet to ~300 chars | None — fetch the file with `getFileContent` for full context | | `listDiscussions` | Returns 20 discussions per call, cursor-paginated | Raise `perPage`, or pass the returned `endCursor` as `after` | | `getWorkflowJobLogs` | Returns the last 200 log lines with per-line timestamps stripped | Raise `maxLines` (up to 2000) when the error is higher up | | `listPullRequestReviewThreads` | Returns unresolved threads only, comment bodies truncated (~500 chars), cursor-paginated | `status: 'all'` for resolved threads; `detail: 'full'` for complete bodies; pass `endCursor` as `after` | -| `listNotifications` | Returns 20 unread threads per call (max 50) | `all: true` to include read threads; raise `perPage` | +| `listNotifications` | Returns 20 unread threads per call (max 50) | `all: true` to include read threads; raise `perPage`; when `hasMore`, pass `nextPage` | ## Example: code review bootstrap diff --git a/apps/docs/content/docs/5.api/1.tools-catalog.md b/apps/docs/content/docs/5.api/1.tools-catalog.md index f3bdd5c..e0ba95a 100644 --- a/apps/docs/content/docs/5.api/1.tools-catalog.md +++ b/apps/docs/content/docs/5.api/1.tools-catalog.md @@ -52,7 +52,7 @@ Available in all presets. These tools manage repositories, branches, and file co | `getRepository` | read repository metadata (name, description, stars, language) | No | | `listBranches` | list branches and their HEAD commits | No | | `getFileContent` | read a file at a path/ref; prefer `startLine`/`endLine` or `maxLines` for large files | No | -| `getRepositoryTree` | list the file and directory structure of a repository at a given ref | No | +| `getRepositoryTree` | list the file and directory structure at a given ref (prefer a `path` prefix over `recursive: true`) | No | | `createBranch` | create a new branch from an existing branch or commit SHA | Yes | | `deleteBranch` | permanently delete a branch | Yes | | `forkRepository` | fork a repository to your account or an organization | Yes | @@ -202,7 +202,7 @@ Available in all presets: | Tool | Capability | Write | |---|---|---| -| `listCommits` | list commit history for a branch | No | +| `listCommits` | list commit history (filter with `path` / `author` / `since` / `until`; when `hasMore`, pass `nextPage`) | No | | `getCommit` | read a single commit with file stats (patches omitted by default; set `includePatch` for diffs) | No | | `getBlame` | line-level git blame for a file (GraphQL) | No | | `compareCommits` | compare two branches, tags, or commits: ahead/behind counts, commits in between, and files that differ (patches omitted by default) | No | @@ -210,9 +210,9 @@ Available in all presets: | `searchRepositories` | search repositories by query | No | | `searchIssues` | search issues and pull requests using qualifiers like `is:open` or `type:pr` | No | -## Fetch beyond one page +## Paged list results -List tools (`listCommits`, `listPullRequests`, `listIssues`, `listWorkflowRuns`, `listCheckRuns`, `listReleases`) accept an optional `maxPages` alongside `perPage`. Omit it to fetch a single page as before; set it to sequentially fetch and combine up to that many pages, stopping early once a page comes back short. This lets the model pull a full history in one call instead of paging manually across several tool calls. +REST list tools return `{ items, hasMore, page, perPage, nextPage? }` — or add those paging fields next to existing keys (`checkRuns`, `runs`, `workflows`, `jobs`). When `hasMore` is true, call again with `nextPage` (not the same `page`), or set `maxPages` to combine sequential pages in one call. Filter `listCommits` with `path` / `author` / `since` / `until` instead of walking the full history. `getRepositoryTree` accepts a `path` prefix; prefer that over `recursive: true`. ## Identify write operations diff --git a/apps/docs/content/docs/5.api/2.reference.md b/apps/docs/content/docs/5.api/2.reference.md index 9529002..2763938 100644 --- a/apps/docs/content/docs/5.api/2.reference.md +++ b/apps/docs/content/docs/5.api/2.reference.md @@ -158,7 +158,7 @@ Core properties (`execute`, `inputSchema`, `outputSchema`) cannot be overridden. ### Rate-limit metadata -Object-shaped tool results include a `rateLimit` field from the last GitHub response (`x-ratelimit-remaining`, `x-ratelimit-limit`, `x-ratelimit-reset`, `x-ratelimit-resource`, and `retry-after` when present). Array-shaped results (`listIssues`, `listPullRequests`, …) are unchanged. The field is stripped before the model sees the output (`toModelOutput`); hooks, channels, and the chat UI still receive it. +Object-shaped tool results include a `rateLimit` field from the last GitHub response (`x-ratelimit-remaining`, `x-ratelimit-limit`, `x-ratelimit-reset`, `x-ratelimit-resource`, and `retry-after` when present). REST list tools now return objects (`{ items, hasMore, … }` or keyed collections), so they carry `rateLimit` too. The field is stripped before the model sees the output (`toModelOutput`); hooks, channels, and the chat UI still receive it. ```ts [rate-limit.ts] import type { GithubRateLimit } from '@github-tools/sdk' diff --git a/apps/docs/skills/github-tools-agents/SKILL.md b/apps/docs/skills/github-tools-agents/SKILL.md index ca63f24..95ff796 100644 --- a/apps/docs/skills/github-tools-agents/SKILL.md +++ b/apps/docs/skills/github-tools-agents/SKILL.md @@ -125,7 +125,7 @@ Array presets merge: `preset: ['code-review', 'issue-triage']`. Start with the s ## Working context -Pass `context: { owner, repo, pullNumber?, issueNumber?, ref? }` to `createGithubTools` / `createGithubAgent` / `createDurableGithubAgent` to default those fields on tool inputs and inject them into the agent system prompt. Prefer composite tools (`getPullRequestContext`, `getIssueContext`, `getReleaseContext`, `getCiFailureContext`) for multi-part reads — call follow-up reads in the same step when possible. Diff patches are omitted by default — set `includePatch: true` (optionally with `filenames`) when you need specific diffs. Bodies are truncated by default (`detail: 'summary'`). `getIssueContext` returns `labelNames` (strings) rather than full label objects. Prefer `getFileContent` with `startLine`/`endLine` or `maxLines` for large files. `getWorkflowJobLogs` returns the last 200 log lines with timestamps stripped — raise `maxLines` (up to 2000) only when needed. `listPullRequestReviewThreads` returns unresolved threads only by default with truncated comment bodies. Object-shaped execute results include `rateLimit` (`remaining` / `limit` / `reset` / `resource`); it is stripped from the model-facing output. Array-shaped list tools do not carry it. On 403/429 the error text includes remaining/reset. +Pass `context: { owner, repo, pullNumber?, issueNumber?, ref? }` to `createGithubTools` / `createGithubAgent` / `createDurableGithubAgent` to default those fields on tool inputs and inject them into the agent system prompt. Prefer composite tools (`getPullRequestContext`, `getIssueContext`, `getReleaseContext`, `getCiFailureContext`) for multi-part reads — call follow-up reads in the same step when possible. Diff patches are omitted by default — set `includePatch: true` (optionally with `filenames`) when you need specific diffs. Bodies are truncated by default (`detail: 'summary'`). `getIssueContext` returns `labelNames` (strings) rather than full label objects. Prefer `getFileContent` with `startLine`/`endLine` or `maxLines` for large files. `getWorkflowJobLogs` returns the last 200 log lines with timestamps stripped — raise `maxLines` (up to 2000) only when needed. `listPullRequestReviewThreads` returns unresolved threads only by default with truncated comment bodies. REST list tools return `{ items, hasMore, page, nextPage }` (or add those fields next to `checkRuns` / `runs`); when `hasMore`, call with `nextPage` or raise `maxPages` — never the same page. Filter `listCommits` with `path` / `author` / `since` / `until`. Prefer a `path` prefix on `getRepositoryTree` over `recursive: true`. Object-shaped execute results include `rateLimit` (`remaining` / `limit` / `reset` / `resource`); it is stripped from the model-facing output. On 403/429 the error text includes remaining/reset. ## Write safety diff --git a/packages/github-tools/README.md b/packages/github-tools/README.md index b029ac2..09df04a 100644 --- a/packages/github-tools/README.md +++ b/packages/github-tools/README.md @@ -188,7 +188,7 @@ Core properties (`execute`, `inputSchema`, `outputSchema`) cannot be overridden. ## Rate-limit metadata -Object-shaped tool results include a `rateLimit` field from the last GitHub response. Array-shaped results are unchanged. The field is stripped before the model sees the output; hooks, channels, and UIs still receive it. +Object-shaped tool results include a `rateLimit` field from the last GitHub response. REST list tools now return objects, so they carry it too. The field is stripped before the model sees the output; hooks, channels, and UIs still receive it. ```ts import type { GithubRateLimit } from '@github-tools/sdk' @@ -471,16 +471,16 @@ New agents should use the [eve extension](#eve-extension) above; see [`examples/ ## Available Tools -List tools (`listCommits`, `listPullRequests`, `listIssues`, `listWorkflowRuns`, `listCheckRuns`, `listReleases`) accept an optional `maxPages` alongside `perPage`. Set it to sequentially fetch and combine up to that many pages in one call, stopping early once a page comes back short. +List tools return `{ items, hasMore, page, perPage, nextPage? }` (or add those fields next to `checkRuns` / `runs` / …). When `hasMore`, call with `nextPage` or set `maxPages` to combine sequential pages in one call — do not repeat the same page. Filter `listCommits` with `path` / `author` / `since` / `until`. `getRepositoryTree` accepts a `path` prefix. ### Repository | Tool | Description | |---|---| | `getRepository` | Get repository metadata (stars, language, default branch, …) | -| `listBranches` | List branches | +| `listBranches` | List branches (`hasMore` / `nextPage` when there are more) | | `getFileContent` | Read a file or directory listing (prefer `startLine`/`endLine` or `maxLines` for large files) | -| `getRepositoryTree` | List the file and directory structure at a given ref | +| `getRepositoryTree` | List the file and directory structure at a given ref (prefer a `path` prefix over `recursive: true`) | | `createBranch` | Create a new branch from an existing branch or commit SHA | | `deleteBranch` | Permanently delete a branch | | `forkRepository` | Fork a repository to a user or organization | @@ -606,7 +606,7 @@ Pull request conversations share the issue numbering, so the issue-level tools w | Tool | Description | |---|---| -| `listCommits` | List commits, optionally filtered by file path, author, or date range | +| `listCommits` | List commits, optionally filtered by file path, author, or date range. When `hasMore`, pass `nextPage` | | `getCommit` | Get a commit's full details including changed files and diffs | | `getBlame` | Line-level git blame for a file (GitHub GraphQL) | | `compareCommits` | Compare two branches, tags, or commits: ahead/behind counts, commits in between, and files that differ | diff --git a/packages/github-tools/src/agents.ts b/packages/github-tools/src/agents.ts index b45b098..b1f82b6 100644 --- a/packages/github-tools/src/agents.ts +++ b/packages/github-tools/src/agents.ts @@ -8,7 +8,8 @@ import { formatContextInstructions, type GithubToolsContext } from './core/conte const SHARED_RULES = `When a tool execution is denied by the user, do not retry it. Briefly acknowledge the decision and move on. Call independent read tools in the same step when you already know the arguments — never serialize reads that could run in parallel. -Bodies default to detail summary; patches default to includePatch false; prefer getFileContent with startLine/endLine or maxLines for large files.` +Bodies default to detail summary; patches default to includePatch false; prefer getFileContent with startLine/endLine or maxLines for large files. +Paged lists return { items, hasMore, page, nextPage }. When hasMore, call with nextPage or raise maxPages — never the same page. Prefer path/author/since/until on listCommits instead of walking history. Prefer a path prefix on getRepositoryTree over recursive true.` const DEFAULT_INSTRUCTIONS = `You are a helpful GitHub assistant. You can read and explore repositories, issues, pull requests, discussions, commits, code, gists, and workflows. You can also create issues, pull requests, comments, gists, reactions, trigger workflows, and update files when asked. diff --git a/packages/github-tools/src/core/bundles.ts b/packages/github-tools/src/core/bundles.ts index 23dbc6f..e4a7a1d 100644 --- a/packages/github-tools/src/core/bundles.ts +++ b/packages/github-tools/src/core/bundles.ts @@ -28,7 +28,7 @@ export const getPullRequestContextInputSchema = z.object({ detail: detailSchema, }) -export const getPullRequestContextDescription = 'Fetch pull request details plus files, reviews, and optional CI checks in one call — prefer this over separate getPullRequest / listPullRequestFiles / listPullRequestReviews calls' +export const getPullRequestContextDescription = 'Fetch pull request details plus files, reviews, and optional CI checks in one call — prefer this over separate getPullRequest / listPullRequestFiles / listPullRequestReviews calls. filesHasMore / reviewsHasMore mean more pages exist on those lists.' export async function getPullRequestContextCore({ token, @@ -80,8 +80,8 @@ export async function getPullRequestContextCore({ return withComposedRateLimit({ pullRequest, - ...files !== undefined ? { files } : {}, - ...reviews !== undefined ? { reviews } : {}, + ...files !== undefined ? { files: files.items, filesHasMore: files.hasMore } : {}, + ...reviews !== undefined ? { reviews: reviews.items, reviewsHasMore: reviews.hasMore } : {}, ...checks !== undefined ? { checks } : {}, }) } @@ -101,7 +101,7 @@ export const getIssueContextInputSchema = z.object({ .describe('full returns the complete body (default for this one-shot tool); summary truncates to ~500 chars'), }) -export const getIssueContextDescription = 'Fetch an issue plus available label names and recent comments in one call — prefer this over separate getIssue / listLabels / comment calls when triaging. Call once; do not re-fetch the same issue.' +export const getIssueContextDescription = 'Fetch an issue plus available label names and recent comments in one call — prefer this over separate getIssue / listLabels / comment calls when triaging. Call once; do not re-fetch the same issue. commentsHasMore means more comments exist — use listIssueComments with nextPage.' export async function getIssueContextCore({ token, @@ -143,8 +143,8 @@ export async function getIssueContextCore({ return withComposedRateLimit({ issue, // Names only — full label objects (color/description) dominate triage payloads on large repos - ...labels !== undefined ? { labelNames: labels.map(label => label.name) } : {}, - ...comments !== undefined ? { comments } : {}, + ...labels !== undefined ? { labelNames: labels.items.map(label => label.name) } : {}, + ...comments !== undefined ? { comments: comments.items, commentsHasMore: comments.hasMore } : {}, }) } @@ -180,7 +180,7 @@ export async function getReleaseContextCore({ ? await getReleaseCore({ token, owner, repo, releaseId, detail }) : await getLatestReleaseCore({ token, owner, repo, detail }) - let previous: Awaited>[number] | undefined + let previous: Awaited>['items'][number] | undefined if (includePrevious || includeCompare) { const releases = await listReleasesCore({ token, @@ -190,7 +190,7 @@ export async function getReleaseContextCore({ maxPages: 1, detail, }) - previous = releases.find(r => r.id !== release.id && !r.draft) + previous = releases.items.find(r => r.id !== release.id && !r.draft) } let comparison: Awaited> | undefined diff --git a/packages/github-tools/src/core/checks.ts b/packages/github-tools/src/core/checks.ts index b9a5c74..0a8afd8 100644 --- a/packages/github-tools/src/core/checks.ts +++ b/packages/github-tools/src/core/checks.ts @@ -1,28 +1,29 @@ import { z } from 'zod' import { withOctokit } from '../client' -import { fetchAllPages, maxPagesSchema } from './pagination' +import { fetchAllPages, hasMoreByTotal, maxPagesSchema, pageSchema, pagingFields } from './pagination' export const listCheckRunsInputSchema = z.object({ owner: z.string().describe('Repository owner'), repo: z.string().describe('Repository name'), ref: z.string().describe('Git ref: branch, tag, or commit SHA'), perPage: z.number().optional().default(30).describe('Number of results to return per page (max 100)'), + page: pageSchema, maxPages: maxPagesSchema, }) -export const listCheckRunsDescription = 'List check runs (Checks API — GitHub Actions and other CI providers) for a commit, branch, or tag' +export const listCheckRunsDescription = 'List check runs (Checks API — GitHub Actions and other CI providers) for a commit, branch, or tag. When hasMore, pass nextPage or raise maxPages — do not repeat the same call.' -export async function listCheckRunsCore({ token, owner, repo, ref, perPage, maxPages }: { token: string, owner: string, repo: string, ref: string, perPage: number, maxPages?: number }) { +export async function listCheckRunsCore({ token, owner, repo, ref, perPage, page = 1, maxPages }: { token: string, owner: string, repo: string, ref: string, perPage: number, page?: number, maxPages?: number }) { return withOctokit(token, async (octokit) => { let totalCount = 0 - const checkRuns = await fetchAllPages(async page => { - const { data } = await octokit.rest.checks.listForRef({ owner, repo, ref, per_page: perPage, page }) + const { items } = await fetchAllPages(async currentPage => { + const { data } = await octokit.rest.checks.listForRef({ owner, repo, ref, per_page: perPage, page: currentPage }) totalCount = data.total_count return data.check_runs - }, perPage, maxPages) + }, perPage, maxPages, page) return { totalCount, - checkRuns: checkRuns.map(run => ({ + checkRuns: items.map(run => ({ id: run.id, name: run.name, status: run.status, @@ -31,6 +32,7 @@ export async function listCheckRunsCore({ token, owner, repo, ref, perPage, maxP startedAt: run.started_at, completedAt: run.completed_at, })), + ...pagingFields(page, perPage, items.length, hasMoreByTotal(page, perPage, items.length, totalCount)), } }) } diff --git a/packages/github-tools/src/core/commits.ts b/packages/github-tools/src/core/commits.ts index a7c399e..11b5d48 100644 --- a/packages/github-tools/src/core/commits.ts +++ b/packages/github-tools/src/core/commits.ts @@ -1,6 +1,6 @@ import { z } from 'zod' import { withOctokit } from '../client' -import { fetchAllPages, maxPagesSchema } from './pagination' +import { fetchAllPages, maxPagesSchema, pageSchema, pagedList } from './pagination' export const BLAME_QUERY = ` query ($owner: String!, $name: String!, $expression: String!, $path: String!) { @@ -71,15 +71,16 @@ export const listCommitsInputSchema = z.object({ since: z.string().optional().describe('Only commits after this date (ISO 8601 format)'), until: z.string().optional().describe('Only commits before this date (ISO 8601 format)'), perPage: z.number().optional().default(30).describe('Number of results to return per page (max 100)'), + page: pageSchema, maxPages: maxPagesSchema, }) export const listCommitsDescription = - 'List commits for a GitHub repository. Filter by file path to see commits that touched a file. For line-by-line attribution at a given ref, use getBlame instead.' + 'List commits for a GitHub repository. Filter with path, author, since, or until instead of paging the full history. When hasMore, pass nextPage or raise maxPages — do not repeat the same call. For line-by-line attribution, use getBlame.' -export async function listCommitsCore({ token, owner, repo, path, sha, author, since, until, perPage, maxPages }: { token: string, owner: string, repo: string, path?: string, sha?: string, author?: string, since?: string, until?: string, perPage: number, maxPages?: number }) { +export async function listCommitsCore({ token, owner, repo, path, sha, author, since, until, perPage, page = 1, maxPages }: { token: string, owner: string, repo: string, path?: string, sha?: string, author?: string, since?: string, until?: string, perPage: number, page?: number, maxPages?: number }) { return withOctokit(token, async (octokit) => { - const commits = await fetchAllPages(async page => { + const { items, hasMore } = await fetchAllPages(async currentPage => { const { data } = await octokit.rest.repos.listCommits({ owner, repo, @@ -89,18 +90,18 @@ export async function listCommitsCore({ token, owner, repo, path, sha, author, s since, until, per_page: perPage, - page, + page: currentPage, }) return data - }, perPage, maxPages) - return commits.map(commit => ({ + }, perPage, maxPages, page) + return pagedList(items.map(commit => ({ sha: commit.sha, message: commit.commit.message, author: commit.commit.author?.name, authorLogin: commit.author?.login, date: commit.commit.author?.date, url: commit.html_url, - })) + })), perPage, page, hasMore) }) } diff --git a/packages/github-tools/src/core/gists.ts b/packages/github-tools/src/core/gists.ts index 669e21c..ebda5ff 100644 --- a/packages/github-tools/src/core/gists.ts +++ b/packages/github-tools/src/core/gists.ts @@ -1,20 +1,21 @@ import { z } from 'zod' import { withOctokit } from '../client' +import { pageSchema, pagedList } from './pagination' export const listGistsInputSchema = z.object({ username: z.string().optional().describe('GitHub username — omit to list your own gists'), perPage: z.number().optional().default(30).describe('Number of results to return (max 100)'), - page: z.number().optional().default(1).describe('Page number for pagination'), + page: pageSchema, }) -export const listGistsDescription = 'List gists for the authenticated user or a specific user' +export const listGistsDescription = 'List gists for the authenticated user or a specific user. When hasMore, pass nextPage — do not repeat the same call.' export async function listGistsCore({ token, username, perPage, page }: { token: string, username?: string, perPage: number, page: number }) { return withOctokit(token, async (octokit) => { const { data } = username ? await octokit.rest.gists.listForUser({ username, per_page: perPage, page }) : await octokit.rest.gists.list({ per_page: perPage, page }) - return data.map(gist => ({ + return pagedList(data.map(gist => ({ id: gist.id, description: gist.description, public: gist.public, @@ -24,7 +25,7 @@ export async function listGistsCore({ token, username, perPage, page }: { token: comments: gist.comments, createdAt: gist.created_at, updatedAt: gist.updated_at, - })) + })), perPage, page, data.length >= perPage) }) } @@ -59,22 +60,22 @@ export async function getGistCore({ token, gistId }: { token: string, gistId: st export const listGistCommentsInputSchema = z.object({ gistId: z.string().describe('Gist ID'), perPage: z.number().optional().default(30).describe('Number of results to return (max 100)'), - page: z.number().optional().default(1).describe('Page number for pagination'), + page: pageSchema, }) -export const listGistCommentsDescription = 'List comments on a gist' +export const listGistCommentsDescription = 'List comments on a gist. When hasMore, pass nextPage — do not repeat the same call.' export async function listGistCommentsCore({ token, gistId, perPage, page }: { token: string, gistId: string, perPage: number, page: number }) { return withOctokit(token, async (octokit) => { const { data } = await octokit.rest.gists.listComments({ gist_id: gistId, per_page: perPage, page }) - return data.map(comment => ({ + return pagedList(data.map(comment => ({ id: comment.id, body: comment.body, author: comment.user?.login, url: comment.url, createdAt: comment.created_at, updatedAt: comment.updated_at, - })) + })), perPage, page, data.length >= perPage) }) } diff --git a/packages/github-tools/src/core/issues.ts b/packages/github-tools/src/core/issues.ts index 893775e..67264d5 100644 --- a/packages/github-tools/src/core/issues.ts +++ b/packages/github-tools/src/core/issues.ts @@ -1,7 +1,7 @@ import { z } from 'zod' import { withOctokit } from '../client' import { applyDetailBody, detailSchema, type DetailLevel } from './detail' -import { fetchAllPages, maxPagesSchema } from './pagination' +import { fetchAllPages, maxPagesSchema, pageSchema, pagedList } from './pagination' export const listIssuesInputSchema = z.object({ owner: z.string().describe('Repository owner'), @@ -9,25 +9,26 @@ export const listIssuesInputSchema = z.object({ state: z.enum(['open', 'closed', 'all']).optional().default('open').describe('Filter by state'), labels: z.string().optional().describe('Comma-separated list of label names to filter by'), perPage: z.number().optional().default(30).describe('Number of results to return per page (max 100)'), + page: pageSchema, maxPages: maxPagesSchema, }) -export const listIssuesDescription = 'List issues for a GitHub repository (excludes pull requests)' +export const listIssuesDescription = 'List issues for a GitHub repository (excludes pull requests). When hasMore, pass nextPage or raise maxPages — do not repeat the same call.' -export async function listIssuesCore({ token, owner, repo, state, labels, perPage, maxPages }: { token: string, owner: string, repo: string, state: 'open' | 'closed' | 'all', labels?: string, perPage: number, maxPages?: number }) { +export async function listIssuesCore({ token, owner, repo, state, labels, perPage, page = 1, maxPages }: { token: string, owner: string, repo: string, state: 'open' | 'closed' | 'all', labels?: string, perPage: number, page?: number, maxPages?: number }) { return withOctokit(token, async (octokit) => { - const issues = await fetchAllPages(async page => { + const { items, hasMore } = await fetchAllPages(async currentPage => { const { data } = await octokit.rest.issues.listForRepo({ owner, repo, state, labels, per_page: perPage, - page, + page: currentPage, }) return data - }, perPage, maxPages) - return issues + }, perPage, maxPages, page) + return pagedList(items .filter(issue => !issue.pull_request) .map(issue => ({ number: issue.number, @@ -38,7 +39,7 @@ export async function listIssuesCore({ token, owner, repo, state, labels, perPag labels: issue.labels.map(l => (typeof l === 'string' ? l : l.name)), createdAt: issue.created_at, updatedAt: issue.updated_at, - })) + })), perPage, page, hasMore) }) } @@ -76,23 +77,23 @@ export const listIssueCommentsInputSchema = z.object({ repo: z.string().describe('Repository name'), issueNumber: z.number().describe('Issue number'), perPage: z.number().optional().default(30).describe('Number of comments to return (max 100)'), - page: z.number().optional().default(1).describe('Page number for pagination'), + page: pageSchema, detail: detailSchema, }) -export const listIssueCommentsDescription = 'List comments on a GitHub issue. Bodies are truncated by default (detail: summary)' +export const listIssueCommentsDescription = 'List comments on a GitHub issue. Bodies are truncated by default (detail: summary). When hasMore, pass nextPage — do not repeat the same call.' export async function listIssueCommentsCore({ token, owner, repo, issueNumber, perPage, page, detail = 'summary' }: { token: string, owner: string, repo: string, issueNumber: number, perPage: number, page: number, detail?: DetailLevel }) { return withOctokit(token, async (octokit) => { const { data } = await octokit.rest.issues.listComments({ owner, repo, issue_number: issueNumber, per_page: perPage, page }) - return data.map(comment => ({ + return pagedList(data.map(comment => ({ id: comment.id, url: comment.html_url, body: applyDetailBody(comment.body, detail), author: comment.user?.login, createdAt: comment.created_at, updatedAt: comment.updated_at, - })) + })), perPage, page, data.length >= perPage) }) } @@ -270,19 +271,19 @@ export const listLabelsInputSchema = z.object({ owner: z.string().describe('Repository owner'), repo: z.string().describe('Repository name'), perPage: z.number().optional().default(30).describe('Number of results to return (max 100)'), - page: z.number().optional().default(1).describe('Page number for pagination'), + page: pageSchema, }) -export const listLabelsDescription = 'List labels available in a GitHub repository' +export const listLabelsDescription = 'List labels available in a GitHub repository. When hasMore, pass nextPage — do not repeat the same call.' export async function listLabelsCore({ token, owner, repo, perPage, page }: { token: string, owner: string, repo: string, perPage: number, page: number }) { return withOctokit(token, async (octokit) => { const { data } = await octokit.rest.issues.listLabelsForRepo({ owner, repo, per_page: perPage, page }) - return data.map(label => ({ + return pagedList(data.map(label => ({ name: label.name, color: label.color, description: label.description, - })) + })), perPage, page, data.length >= perPage) }) } diff --git a/packages/github-tools/src/core/model-output.test.ts b/packages/github-tools/src/core/model-output.test.ts new file mode 100644 index 0000000..299fce1 --- /dev/null +++ b/packages/github-tools/src/core/model-output.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest' +import { + getCommitToModelOutput, + getFileContentToModelOutput, + getRepositoryTreeToModelOutput, + listPullRequestFilesToModelOutput, +} from './model-output' + +const options = { toolCallId: 'call', input: {} } + +describe('listPullRequestFilesToModelOutput', () => { + it('keeps paging fields and truncates patches', () => { + const patch = 'x'.repeat(5000) + const result = listPullRequestFilesToModelOutput({ + ...options, + output: { + items: [{ filename: 'a.ts', status: 'modified', additions: 1, deletions: 0, changes: 1, patch }], + hasMore: true, + page: 1, + perPage: 30, + nextPage: 2, + }, + }) + const value = result.value as { items: Array<{ patch: string }>, hasMore: boolean, nextPage?: number } + expect(value.hasMore).toBe(true) + expect(value.nextPage).toBe(2) + expect(value.items[0]!.patch.length).toBeLessThan(patch.length) + expect(value.items[0]!.patch).toContain('[truncated:') + }) +}) + +describe('getRepositoryTreeToModelOutput', () => { + it('caps entries and sets truncated', () => { + const entries = Array.from({ length: 250 }, (_, i) => ({ path: `f${i}.ts`, type: 'blob' })) + const result = getRepositoryTreeToModelOutput({ + ...options, + output: { sha: 'abc', truncated: false, entries }, + }) + const value = result.value as { entries: unknown[], truncated: boolean, entriesOmitted: number } + expect(value.entries).toHaveLength(200) + expect(value.truncated).toBe(true) + expect(value.entriesOmitted).toBe(50) + }) +}) + +describe('getFileContentToModelOutput', () => { + it('caps directory listings', () => { + const entries = Array.from({ length: 250 }, (_, i) => ({ name: `f${i}`, type: 'file', path: `f${i}` })) + const result = getFileContentToModelOutput({ + ...options, + output: { type: 'directory', entries }, + }) + const value = result.value as { entries: unknown[], truncated: boolean, entriesOmitted: number } + expect(value.entries).toHaveLength(200) + expect(value.truncated).toBe(true) + expect(value.entriesOmitted).toBe(50) + }) +}) + +describe('getCommitToModelOutput', () => { + it('caps files and reports filesOmitted', () => { + const files = Array.from({ length: 90 }, (_, i) => ({ + filename: `f${i}.ts`, + status: 'modified', + additions: 1, + deletions: 0, + })) + const result = getCommitToModelOutput({ + ...options, + output: { sha: 'abc', message: 'm', url: 'https://example.com', stats: null, files }, + }) + const value = result.value as { files: unknown[], filesOmitted: number } + expect(value.files).toHaveLength(80) + expect(value.filesOmitted).toBe(10) + }) +}) diff --git a/packages/github-tools/src/core/model-output.ts b/packages/github-tools/src/core/model-output.ts index ede8f39..1893a43 100644 --- a/packages/github-tools/src/core/model-output.ts +++ b/packages/github-tools/src/core/model-output.ts @@ -1,5 +1,7 @@ const MAX_PATCH_LENGTH = 4000 const MAX_CONTENT_LENGTH = 20000 +const MAX_MODEL_TREE_ENTRIES = 200 +const MAX_MODEL_DIFF_FILES = 80 function truncateText(text: string, maxLength: number): string { if (text.length <= maxLength) return text @@ -13,20 +15,38 @@ function truncatePatchFields(files: T[]): T[] { })) } +function capDiffFiles(files: T[] | undefined) { + if (!files) return { files, filesOmitted: 0 } + const truncated = truncatePatchFields(files) + if (truncated.length <= MAX_MODEL_DIFF_FILES) return { files: truncated, filesOmitted: 0 } + return { + files: truncated.slice(0, MAX_MODEL_DIFF_FILES), + filesOmitted: truncated.length - MAX_MODEL_DIFF_FILES, + } +} + type ToModelOutputOptions = { toolCallId: string input: unknown output: unknown } -type ListPullRequestFilesOutput = Array<{ +type ListPullRequestFile = { filename: string status: string additions: number deletions: number changes: number patch?: string -}> +} + +type ListPullRequestFilesOutput = { + items: ListPullRequestFile[] + hasMore: boolean + page: number + perPage: number + nextPage?: number +} type GetCommitOutput = { sha: string @@ -61,20 +81,25 @@ type GetFileContentOutput = } export function listPullRequestFilesToModelOutput({ output }: ToModelOutputOptions) { - const files = output as ListPullRequestFilesOutput + const result = output as ListPullRequestFilesOutput return { type: 'json' as const, - value: truncatePatchFields(files), + value: { + ...result, + items: truncatePatchFields(result.items), + }, } } export function getCommitToModelOutput({ output }: ToModelOutputOptions) { const commit = output as GetCommitOutput + const { files, filesOmitted } = capDiffFiles(commit.files) return { type: 'json' as const, value: { ...commit, - files: commit.files ? truncatePatchFields(commit.files) : commit.files, + files, + ...filesOmitted > 0 ? { filesOmitted } : {}, }, } } @@ -97,17 +122,30 @@ type CompareCommitsOutput = { export function compareCommitsToModelOutput({ output }: ToModelOutputOptions) { const comparison = output as CompareCommitsOutput + const { files, filesOmitted } = capDiffFiles(comparison.files) return { type: 'json' as const, value: { ...comparison, - files: comparison.files ? truncatePatchFields(comparison.files) : comparison.files, + files, + ...filesOmitted > 0 ? { filesOmitted } : {}, }, } } export function getFileContentToModelOutput({ output }: ToModelOutputOptions) { const result = output as GetFileContentOutput + if (result.type === 'directory' && 'entries' in result && result.entries.length > MAX_MODEL_TREE_ENTRIES) { + return { + type: 'json' as const, + value: { + ...result, + truncated: true, + entries: result.entries.slice(0, MAX_MODEL_TREE_ENTRIES), + entriesOmitted: result.entries.length - MAX_MODEL_TREE_ENTRIES, + }, + } + } if ('content' in result && result.content.length > MAX_CONTENT_LENGTH) { return { type: 'json' as const, @@ -120,6 +158,29 @@ export function getFileContentToModelOutput({ output }: ToModelOutputOptions) { return { type: 'json' as const, value: result } } +type GetRepositoryTreeOutput = { + sha: string + truncated: boolean + path?: string + entries: Array<{ path?: string, type?: string, size?: number, sha?: string }> +} + +export function getRepositoryTreeToModelOutput({ output }: ToModelOutputOptions) { + const result = output as GetRepositoryTreeOutput + if (result.entries.length <= MAX_MODEL_TREE_ENTRIES) { + return { type: 'json' as const, value: result } + } + return { + type: 'json' as const, + value: { + ...result, + truncated: true, + entries: result.entries.slice(0, MAX_MODEL_TREE_ENTRIES), + entriesOmitted: result.entries.length - MAX_MODEL_TREE_ENTRIES, + }, + } +} + type GetPullRequestContextOutput = { pullRequest: { number: number @@ -141,7 +202,8 @@ type GetPullRequestContextOutput = { updatedAt: string mergedAt: string | null } - files?: ListPullRequestFilesOutput + files?: ListPullRequestFile[] + filesHasMore?: boolean reviews?: Array<{ id: number state: string @@ -178,11 +240,13 @@ type GetPullRequestContextOutput = { export function getPullRequestContextToModelOutput({ output }: ToModelOutputOptions) { const result = output as GetPullRequestContextOutput + const { files, filesOmitted } = capDiffFiles(result.files) return { type: 'json' as const, value: { ...result, - files: result.files ? truncatePatchFields(result.files) : result.files, + files, + ...filesOmitted > 0 ? { filesOmitted } : {}, }, } } diff --git a/packages/github-tools/src/core/notifications.ts b/packages/github-tools/src/core/notifications.ts index 5ddb925..d00cfdc 100644 --- a/packages/github-tools/src/core/notifications.ts +++ b/packages/github-tools/src/core/notifications.ts @@ -1,14 +1,15 @@ import { z } from 'zod' import { withOctokit } from '../client' +import { pageSchema, pagedList } from './pagination' export const listNotificationsInputSchema = z.object({ all: z.boolean().optional().default(false).describe('Include notifications already marked as read'), participating: z.boolean().optional().default(false).describe('Only notifications where the authenticated user is directly participating or mentioned'), perPage: z.number().optional().default(20).describe('Number of notifications to return (max 50)'), - page: z.number().optional().default(1).describe('Page number for pagination'), + page: pageSchema, }) -export const listNotificationsDescription = 'List notification threads for the authenticated user. Unread only by default — set all true to include read threads. Requires a token with notifications access' +export const listNotificationsDescription = 'List notification threads for the authenticated user. Unread only by default — set all true to include read threads. Requires a token with notifications access. When hasMore, pass nextPage — do not repeat the same call.' export async function listNotificationsCore({ token, all, participating, perPage, page }: { token: string, all: boolean, participating: boolean, perPage: number, page: number }) { return withOctokit(token, async (octokit) => { @@ -18,7 +19,7 @@ export async function listNotificationsCore({ token, all, participating, perPage per_page: perPage, page, }) - return data.map(thread => ({ + return pagedList(data.map(thread => ({ threadId: thread.id, repository: thread.repository.full_name, subject: { @@ -29,7 +30,7 @@ export async function listNotificationsCore({ token, all, participating, perPage reason: thread.reason, unread: thread.unread, updatedAt: thread.updated_at, - })) + })), perPage, page, data.length >= perPage) }) } diff --git a/packages/github-tools/src/core/pagination.test.ts b/packages/github-tools/src/core/pagination.test.ts new file mode 100644 index 0000000..12f8a0f --- /dev/null +++ b/packages/github-tools/src/core/pagination.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest' +import { fetchAllPages, hasMoreByTotal, pagedList } from './pagination' + +describe('fetchAllPages', () => { + it('returns one page and hasMore when the page is full', async () => { + const result = await fetchAllPages(async () => [1, 2, 3], 3) + expect(result).toEqual({ items: [1, 2, 3], hasMore: true }) + }) + + it('returns hasMore false when the page is short', async () => { + const result = await fetchAllPages(async () => [1, 2], 3) + expect(result).toEqual({ items: [1, 2], hasMore: false }) + }) + + it('combines pages and stops on a short page', async () => { + const pages = [[1, 2], [3]] + const result = await fetchAllPages(async page => pages[page - 1] ?? [], 2, 5) + expect(result).toEqual({ items: [1, 2, 3], hasMore: false }) + }) + + it('sets hasMore when maxPages is exhausted on full pages', async () => { + const result = await fetchAllPages(async page => [page, page], 2, 2) + expect(result).toEqual({ items: [1, 1, 2, 2], hasMore: true }) + }) + + it('starts at startPage', async () => { + const seen: number[] = [] + await fetchAllPages(async page => { + seen.push(page) + return [page] + }, 10, 1, 4) + expect(seen).toEqual([4]) + }) +}) + +describe('pagedList', () => { + it('packs items with pagination fields', () => { + expect(pagedList(['a'], 30, 2, true)).toEqual({ + items: ['a'], + hasMore: true, + page: 2, + perPage: 30, + nextPage: 3, + }) + }) + + it('omits nextPage when hasMore is false', () => { + expect(pagedList(['a'], 30, 1, false)).toEqual({ + items: ['a'], + hasMore: false, + page: 1, + perPage: 30, + }) + }) + + it('sets nextPage past combined pages', () => { + expect(pagedList([1, 2, 3, 4], 2, 1, true).nextPage).toBe(3) + }) +}) + +describe('hasMoreByTotal', () => { + it('is false on the last short page', () => { + expect(hasMoreByTotal(4, 30, 10, 100)).toBe(false) + }) + + it('is true when combined pages still leave remainder', () => { + expect(hasMoreByTotal(1, 30, 60, 100)).toBe(true) + }) +}) diff --git a/packages/github-tools/src/core/pagination.ts b/packages/github-tools/src/core/pagination.ts index 088b637..e0cf1e6 100644 --- a/packages/github-tools/src/core/pagination.ts +++ b/packages/github-tools/src/core/pagination.ts @@ -1,26 +1,65 @@ import { z } from 'zod' +export const pageSchema = z.number().int().positive().optional().default(1) + .describe('1-based page. When hasMore is true, call again with nextPage (or page + 1). Do not repeat the same page.') + export const maxPagesSchema = z.number().int().positive().max(20).optional() - .describe('Automatically fetch and combine up to this many pages on top of perPage. Omit to fetch a single page.') + .describe('Fetch and combine up to this many pages in one call (max 20). Prefer this over many page=N calls. Omit to fetch a single page. If hasMore, the next start page is nextPage, not page + 1.') + +export type PagingFields = { + hasMore: boolean + page: number + perPage: number + nextPage?: number +} + +export type PagedList = { items: T[] } & PagingFields + +export function pagingFields(page: number, perPage: number, itemCount: number, hasMore: boolean): PagingFields { + return { + hasMore, + page, + perPage, + ...hasMore ? { nextPage: page + Math.max(1, Math.ceil(itemCount / perPage)) } : {}, + } +} + +/** True when the returned window does not yet cover `totalCount`. */ +export function hasMoreByTotal(page: number, perPage: number, itemCount: number, totalCount: number): boolean { + return (page - 1) * perPage + itemCount < totalCount +} + +export function pagedList(items: T[], perPage: number, page: number, hasMore: boolean): PagedList { + return { items, ...pagingFields(page, perPage, items.length, hasMore) } +} /** * Fetches a single page (`startPage`, default 1) by default. When `maxPages` * is set, fetches pages sequentially starting at `startPage` and stops early * once a page returns fewer than `perPage` items (the last page). + * `hasMore` is true when the last fetched page was full. */ export async function fetchAllPages( fetchPage: (page: number) => Promise, perPage: number, maxPages?: number, startPage = 1, -): Promise { - if (!maxPages || maxPages <= 1) return fetchPage(startPage) +): Promise<{ items: T[], hasMore: boolean }> { + if (!maxPages || maxPages <= 1) { + const items = await fetchPage(startPage) + return { items, hasMore: items.length >= perPage } + } - const results: T[] = [] + const items: T[] = [] + let hasMore = false for (let i = 0; i < maxPages; i++) { - const items = await fetchPage(startPage + i) - results.push(...items) - if (items.length < perPage) break + const pageItems = await fetchPage(startPage + i) + items.push(...pageItems) + if (pageItems.length < perPage) { + hasMore = false + break + } + hasMore = true } - return results + return { items, hasMore } } diff --git a/packages/github-tools/src/core/pull-requests.ts b/packages/github-tools/src/core/pull-requests.ts index 9df5435..f5f942f 100644 --- a/packages/github-tools/src/core/pull-requests.ts +++ b/packages/github-tools/src/core/pull-requests.ts @@ -2,7 +2,7 @@ import { z } from 'zod' import { withOctokit } from '../client' import type { CommitIdentity } from '../types' import { applyDetailBody, detailSchema, type DetailLevel } from './detail' -import { fetchAllPages, maxPagesSchema } from './pagination' +import { fetchAllPages, maxPagesSchema, pageSchema, pagedList } from './pagination' import { composeCommitMessage } from './repository' export const listPullRequestsInputSchema = z.object({ @@ -10,18 +10,19 @@ export const listPullRequestsInputSchema = z.object({ repo: z.string().describe('Repository name'), state: z.enum(['open', 'closed', 'all']).optional().default('open').describe('Filter by state'), perPage: z.number().optional().default(30).describe('Number of results to return per page (max 100)'), + page: pageSchema, maxPages: maxPagesSchema, }) -export const listPullRequestsDescription = 'List pull requests for a GitHub repository' +export const listPullRequestsDescription = 'List pull requests for a GitHub repository. When hasMore, pass nextPage or raise maxPages — do not repeat the same call.' -export async function listPullRequestsCore({ token, owner, repo, state, perPage, maxPages }: { token: string, owner: string, repo: string, state: 'open' | 'closed' | 'all', perPage: number, maxPages?: number }) { +export async function listPullRequestsCore({ token, owner, repo, state, perPage, page = 1, maxPages }: { token: string, owner: string, repo: string, state: 'open' | 'closed' | 'all', perPage: number, page?: number, maxPages?: number }) { return withOctokit(token, async (octokit) => { - const pullRequests = await fetchAllPages(async page => { - const { data } = await octokit.rest.pulls.list({ owner, repo, state, per_page: perPage, page }) + const { items, hasMore } = await fetchAllPages(async currentPage => { + const { data } = await octokit.rest.pulls.list({ owner, repo, state, per_page: perPage, page: currentPage }) return data - }, perPage, maxPages) - return pullRequests.map(pr => ({ + }, perPage, maxPages, page) + return pagedList(items.map(pr => ({ number: pr.number, title: pr.title, state: pr.state, @@ -32,7 +33,7 @@ export async function listPullRequestsCore({ token, owner, repo, state, perPage, draft: pr.draft, createdAt: pr.created_at, updatedAt: pr.updated_at, - })) + })), perPage, page, hasMore) }) } @@ -269,16 +270,16 @@ export const listPullRequestFilesInputSchema = z.object({ includePatch: z.boolean().optional().default(false).describe('Include diff patches (token-heavy). Prefer false for an overview, then set true with filenames to fetch specific diffs'), filenames: z.array(z.string()).optional().describe('If set, only return these file paths (useful with includePatch: true for targeted diffs)'), perPage: z.number().optional().default(30).describe('Number of results to return (max 100)'), - page: z.number().optional().default(1).describe('Page number for pagination'), + page: pageSchema, }) -export const listPullRequestFilesDescription = 'List files changed in a pull request with status and stats. Patches are omitted by default — set includePatch true (optionally with filenames) to fetch diffs' +export const listPullRequestFilesDescription = 'List files changed in a pull request with status and stats. Patches are omitted by default — set includePatch true (optionally with filenames) to fetch diffs. When hasMore, pass nextPage — do not repeat the same call.' export async function listPullRequestFilesCore({ token, owner, repo, pullNumber, includePatch, filenames, perPage, page }: { token: string, owner: string, repo: string, pullNumber: number, includePatch: boolean, filenames?: string[], perPage: number, page: number }) { return withOctokit(token, async (octokit) => { const { data } = await octokit.rest.pulls.listFiles({ owner, repo, pull_number: pullNumber, per_page: perPage, page }) const filenameSet = filenames?.length ? new Set(filenames) : null - return data + return pagedList(data .filter(file => !filenameSet || filenameSet.has(file.filename)) .map(file => ({ filename: file.filename, @@ -287,7 +288,7 @@ export async function listPullRequestFilesCore({ token, owner, repo, pullNumber, deletions: file.deletions, changes: file.changes, ...includePatch && file.patch != null ? { patch: file.patch } : {}, - })) + })), perPage, page, data.length >= perPage) }) } @@ -296,22 +297,22 @@ export const listPullRequestReviewsInputSchema = z.object({ repo: z.string().describe('Repository name'), pullNumber: z.number().describe('Pull request number'), perPage: z.number().optional().default(30).describe('Number of results to return (max 100)'), - page: z.number().optional().default(1).describe('Page number for pagination'), + page: pageSchema, }) -export const listPullRequestReviewsDescription = 'List reviews on a pull request (approvals, change requests, and comments)' +export const listPullRequestReviewsDescription = 'List reviews on a pull request (approvals, change requests, and comments). When hasMore, pass nextPage — do not repeat the same call.' export async function listPullRequestReviewsCore({ token, owner, repo, pullNumber, perPage, page }: { token: string, owner: string, repo: string, pullNumber: number, perPage: number, page: number }) { return withOctokit(token, async (octokit) => { const { data } = await octokit.rest.pulls.listReviews({ owner, repo, pull_number: pullNumber, per_page: perPage, page }) - return data.map(review => ({ + return pagedList(data.map(review => ({ id: review.id, state: review.state, body: review.body, author: review.user?.login, url: review.html_url, submittedAt: review.submitted_at, - })) + })), perPage, page, data.length >= perPage) }) } diff --git a/packages/github-tools/src/core/reactions.ts b/packages/github-tools/src/core/reactions.ts index 93a716a..92ce8a4 100644 --- a/packages/github-tools/src/core/reactions.ts +++ b/packages/github-tools/src/core/reactions.ts @@ -1,5 +1,6 @@ import { z } from 'zod' import { withOctokit } from '../client' +import { pageSchema, pagingFields } from './pagination' const REACTION_CONTENTS = ['+1', '-1', 'laugh', 'confused', 'heart', 'hooray', 'rocket', 'eyes'] as const @@ -9,13 +10,14 @@ const reactionContentSchema = z.enum(REACTION_CONTENTS) type ReactionListItem = { content: string, user?: { login: string } | null } -function shapeReactions(reactions: ReactionListItem[]) { +function shapeReactions(reactions: ReactionListItem[], perPage: number, page: number) { const counts: Record = {} for (const reaction of reactions) counts[reaction.content] = (counts[reaction.content] ?? 0) + 1 return { total: reactions.length, counts, reactions: reactions.map(reaction => ({ content: reaction.content, user: reaction.user?.login })), + ...pagingFields(page, perPage, reactions.length, reactions.length >= perPage), } } @@ -25,10 +27,10 @@ export const listIssueReactionsInputSchema = z.object({ issueNumber: z.number().describe('Issue or pull request number — pull request conversations share the issue numbering'), content: reactionContentSchema.optional().describe('Only return reactions of this type'), perPage: z.number().optional().default(30).describe('Number of reactions to return (max 100)'), - page: z.number().optional().default(1).describe('Page number for pagination'), + page: pageSchema, }) -export const listIssueReactionsDescription = 'List reactions on an issue or pull request conversation, with per-emoji counts for the returned page' +export const listIssueReactionsDescription = 'List reactions on an issue or pull request conversation, with per-emoji counts for the returned page. When hasMore, pass nextPage — do not repeat the same call.' export async function listIssueReactionsCore({ token, owner, repo, issueNumber, content, perPage, page }: { token: string, owner: string, repo: string, issueNumber: number, content?: ReactionContent, perPage: number, page: number }) { return withOctokit(token, async (octokit) => { @@ -40,7 +42,7 @@ export async function listIssueReactionsCore({ token, owner, repo, issueNumber, per_page: perPage, page, }) - return shapeReactions(data) + return shapeReactions(data, perPage, page) }) } @@ -77,10 +79,10 @@ export const listCommentReactionsInputSchema = z.object({ commentId: z.number().describe('Issue or pull request comment ID (from getIssueContext or addIssueComment)'), content: reactionContentSchema.optional().describe('Only return reactions of this type'), perPage: z.number().optional().default(30).describe('Number of reactions to return (max 100)'), - page: z.number().optional().default(1).describe('Page number for pagination'), + page: pageSchema, }) -export const listCommentReactionsDescription = 'List reactions on an issue or pull request comment, with per-emoji counts for the returned page' +export const listCommentReactionsDescription = 'List reactions on an issue or pull request comment, with per-emoji counts for the returned page. When hasMore, pass nextPage — do not repeat the same call.' export async function listCommentReactionsCore({ token, owner, repo, commentId, content, perPage, page }: { token: string, owner: string, repo: string, commentId: number, content?: ReactionContent, perPage: number, page: number }) { return withOctokit(token, async (octokit) => { @@ -92,7 +94,7 @@ export async function listCommentReactionsCore({ token, owner, repo, commentId, per_page: perPage, page, }) - return shapeReactions(data) + return shapeReactions(data, perPage, page) }) } diff --git a/packages/github-tools/src/core/releases.ts b/packages/github-tools/src/core/releases.ts index 75193ed..ef93ecc 100644 --- a/packages/github-tools/src/core/releases.ts +++ b/packages/github-tools/src/core/releases.ts @@ -1,25 +1,26 @@ import { z } from 'zod' import { withOctokit } from '../client' import { applyDetailBody, detailSchema, type DetailLevel } from './detail' -import { fetchAllPages, maxPagesSchema } from './pagination' +import { fetchAllPages, maxPagesSchema, pageSchema, pagedList } from './pagination' export const listReleasesInputSchema = z.object({ owner: z.string().describe('Repository owner'), repo: z.string().describe('Repository name'), perPage: z.number().optional().default(30).describe('Number of results to return per page (max 100)'), + page: pageSchema, maxPages: maxPagesSchema, detail: detailSchema, }) -export const listReleasesDescription = 'List releases for a GitHub repository, newest first (includes drafts and prereleases). Bodies truncated by default (detail: summary)' +export const listReleasesDescription = 'List releases for a GitHub repository, newest first (includes drafts and prereleases). Bodies truncated by default (detail: summary). When hasMore, pass nextPage or raise maxPages — do not repeat the same call.' -export async function listReleasesCore({ token, owner, repo, perPage, maxPages, detail = 'summary' }: { token: string, owner: string, repo: string, perPage: number, maxPages?: number, detail?: DetailLevel }) { +export async function listReleasesCore({ token, owner, repo, perPage, page = 1, maxPages, detail = 'summary' }: { token: string, owner: string, repo: string, perPage: number, page?: number, maxPages?: number, detail?: DetailLevel }) { return withOctokit(token, async (octokit) => { - const releases = await fetchAllPages(async page => { - const { data } = await octokit.rest.repos.listReleases({ owner, repo, per_page: perPage, page }) + const { items, hasMore } = await fetchAllPages(async currentPage => { + const { data } = await octokit.rest.repos.listReleases({ owner, repo, per_page: perPage, page: currentPage }) return data - }, perPage, maxPages) - return releases.map(release => ({ + }, perPage, maxPages, page) + return pagedList(items.map(release => ({ id: release.id, tagName: release.tag_name, name: release.name, @@ -30,7 +31,7 @@ export async function listReleasesCore({ token, owner, repo, perPage, maxPages, author: release.author?.login, createdAt: release.created_at, publishedAt: release.published_at, - })) + })), perPage, page, hasMore) }) } diff --git a/packages/github-tools/src/core/repository.ts b/packages/github-tools/src/core/repository.ts index eaa7be8..9507367 100644 --- a/packages/github-tools/src/core/repository.ts +++ b/packages/github-tools/src/core/repository.ts @@ -2,6 +2,7 @@ import { z } from 'zod' import { withOctokit } from '../client' import type { CommitIdentity, Octokit } from '../types' import { gitBlobSha } from './git-blob-sha' +import { pageSchema, pagedList } from './pagination' const CREATE_COMMIT_ON_BRANCH_MUTATION = ` mutation CreateCommitOnBranch($input: CreateCommitOnBranchInput!) { @@ -90,18 +91,19 @@ export const listBranchesInputSchema = z.object({ owner: z.string().describe('Repository owner'), repo: z.string().describe('Repository name'), perPage: z.number().optional().default(30).describe('Number of branches to return (max 100)'), + page: pageSchema, }) -export const listBranchesDescription = 'List branches in a GitHub repository' +export const listBranchesDescription = 'List branches in a GitHub repository. When hasMore, pass nextPage — do not repeat the same call.' -export async function listBranchesCore({ token, owner, repo, perPage }: { token: string, owner: string, repo: string, perPage: number }) { +export async function listBranchesCore({ token, owner, repo, perPage, page = 1 }: { token: string, owner: string, repo: string, perPage: number, page?: number }) { return withOctokit(token, async (octokit) => { - const { data } = await octokit.rest.repos.listBranches({ owner, repo, per_page: perPage }) - return data.map(branch => ({ + const { data } = await octokit.rest.repos.listBranches({ owner, repo, per_page: perPage, page }) + return pagedList(data.map(branch => ({ name: branch.name, sha: branch.commit.sha, protected: branch.protected, - })) + })), perPage, page, data.length >= perPage) }) } @@ -184,24 +186,33 @@ export const getRepositoryTreeInputSchema = z.object({ owner: z.string().describe('Repository owner'), repo: z.string().describe('Repository name'), ref: z.string().optional().describe('Branch, tag, or commit SHA (defaults to the default branch)'), - recursive: z.boolean().optional().default(false).describe('Recursively list the entire tree instead of just the top level'), + path: z.string().optional().describe('Only entries at or under this directory prefix. Prefer this over recursive true on large repos'), + recursive: z.boolean().optional().default(false).describe('Recursively list the entire tree instead of just the top level. Prefer path + recursive false when exploring'), }) -export const getRepositoryTreeDescription = 'List the file and directory structure of a repository at a given ref' +export const getRepositoryTreeDescription = 'List the file and directory structure of a repository at a given ref. Prefer a path prefix over recursive true. If truncated, narrow path instead of recalling the full tree.' -export async function getRepositoryTreeCore({ token, owner, repo, ref, recursive }: { token: string, owner: string, repo: string, ref?: string, recursive: boolean }) { +export async function getRepositoryTreeCore({ token, owner, repo, ref, path, recursive }: { token: string, owner: string, repo: string, ref?: string, path?: string, recursive: boolean }) { return withOctokit(token, async (octokit) => { const treeSha = ref || (await octokit.rest.repos.get({ owner, repo })).data.default_branch const { data } = await octokit.rest.git.getTree({ owner, repo, tree_sha: treeSha, recursive: recursive ? 'true' : undefined }) - return { - sha: data.sha, - truncated: data.truncated, - entries: data.tree.map(entry => ({ + const prefix = path?.replace(/\/+$/, '') + const entries = data.tree + .filter(entry => { + if (!prefix || !entry.path) return !prefix + return entry.path === prefix || entry.path.startsWith(`${prefix}/`) + }) + .map(entry => ({ path: entry.path, type: entry.type, size: entry.size, sha: entry.sha, - })), + })) + return { + sha: data.sha, + truncated: data.truncated, + path: prefix, + entries, } }) } diff --git a/packages/github-tools/src/core/workflows.ts b/packages/github-tools/src/core/workflows.ts index f0ab36a..b0ca006 100644 --- a/packages/github-tools/src/core/workflows.ts +++ b/packages/github-tools/src/core/workflows.ts @@ -1,15 +1,15 @@ import { z } from 'zod' import { withOctokit } from '../client' -import { fetchAllPages, maxPagesSchema } from './pagination' +import { fetchAllPages, hasMoreByTotal, maxPagesSchema, pageSchema, pagingFields } from './pagination' export const listWorkflowsInputSchema = z.object({ owner: z.string().describe('Repository owner'), repo: z.string().describe('Repository name'), perPage: z.number().optional().default(30).describe('Number of results to return (max 100)'), - page: z.number().optional().default(1).describe('Page number for pagination'), + page: pageSchema, }) -export const listWorkflowsDescription = 'List GitHub Actions workflows in a repository' +export const listWorkflowsDescription = 'List GitHub Actions workflows in a repository. When hasMore, pass nextPage — do not repeat the same call.' export async function listWorkflowsCore({ token, owner, repo, perPage, page }: { token: string, owner: string, repo: string, perPage: number, page: number }) { return withOctokit(token, async (octokit) => { @@ -25,6 +25,7 @@ export async function listWorkflowsCore({ token, owner, repo, perPage, page }: { createdAt: wf.created_at, updatedAt: wf.updated_at, })), + ...pagingFields(page, perPage, data.workflows.length, hasMoreByTotal(page, perPage, data.workflows.length, data.total_count)), } }) } @@ -39,16 +40,16 @@ export const listWorkflowRunsInputSchema = z.object({ event: z.string().optional().describe('Event type to filter by (e.g. "push", "pull_request")'), status: z.enum(['completed', 'action_required', 'cancelled', 'failure', 'neutral', 'skipped', 'stale', 'success', 'timed_out', 'in_progress', 'queued', 'requested', 'waiting', 'pending']).optional().describe('Status to filter by'), perPage: z.number().optional().default(30).describe('Number of results to return per page (max 100)'), - page: z.number().optional().default(1).describe('Page number for pagination'), + page: pageSchema, maxPages: maxPagesSchema, }) -export const listWorkflowRunsDescription = 'List workflow runs for a repository, optionally filtered by workflow, branch, status, or event' +export const listWorkflowRunsDescription = 'List workflow runs for a repository, optionally filtered by workflow, branch, status, or event. When hasMore, pass nextPage or raise maxPages — do not repeat the same call.' export async function listWorkflowRunsCore({ token, owner, repo, workflowId, branch, event, status, perPage, page, maxPages }: { token: string, owner: string, repo: string, workflowId?: string | number, branch?: string, event?: string, status?: WorkflowRunStatus, perPage: number, page: number, maxPages?: number }) { return withOctokit(token, async (octokit) => { let totalCount = 0 - const runs = await fetchAllPages(async currentPage => { + const { items } = await fetchAllPages(async currentPage => { const { data } = workflowId ? await octokit.rest.actions.listWorkflowRuns({ owner, repo, workflow_id: workflowId, per_page: perPage, page: currentPage, ...branch && { branch }, ...event && { event }, ...status && { status } }) : await octokit.rest.actions.listWorkflowRunsForRepo({ owner, repo, per_page: perPage, page: currentPage, ...branch && { branch }, ...event && { event }, ...status && { status } }) @@ -58,7 +59,7 @@ export async function listWorkflowRunsCore({ token, owner, repo, workflowId, bra return { totalCount, - runs: runs.map(run => ({ + runs: items.map(run => ({ id: run.id, name: run.name, status: run.status, @@ -72,6 +73,7 @@ export async function listWorkflowRunsCore({ token, owner, repo, workflowId, bra runNumber: run.run_number, runAttempt: run.run_attempt, })), + ...pagingFields(page, perPage, items.length, hasMoreByTotal(page, perPage, items.length, totalCount)), } }) } @@ -112,10 +114,10 @@ export const listWorkflowJobsInputSchema = z.object({ runId: z.number().describe('Workflow run ID'), filter: z.enum(['latest', 'all']).optional().default('latest').describe('Filter by the latest attempt or all attempts'), perPage: z.number().optional().default(30).describe('Number of results to return (max 100)'), - page: z.number().optional().default(1).describe('Page number for pagination'), + page: pageSchema, }) -export const listWorkflowJobsDescription = 'List jobs for a workflow run, including step-level status and timing' +export const listWorkflowJobsDescription = 'List jobs for a workflow run, including step-level status and timing. When hasMore, pass nextPage — do not repeat the same call.' export async function listWorkflowJobsCore({ token, owner, repo, runId, filter, perPage, page }: { token: string, owner: string, repo: string, runId: number, filter: 'latest' | 'all', perPage: number, page: number }) { return withOctokit(token, async (octokit) => { @@ -140,6 +142,7 @@ export async function listWorkflowJobsCore({ token, owner, repo, runId, filter, completedAt: step.completed_at, })), })), + ...pagingFields(page, perPage, data.jobs.length, hasMoreByTotal(page, perPage, data.jobs.length, data.total_count)), } }) } diff --git a/packages/github-tools/src/eve/build.test.ts b/packages/github-tools/src/eve/build.test.ts index 624d033..c760452 100644 --- a/packages/github-tools/src/eve/build.test.ts +++ b/packages/github-tools/src/eve/build.test.ts @@ -139,6 +139,7 @@ describe('createGithubTools eve integration', () => { it('looks up built-in toModelOutput formatters by tool name', () => { expect(hasGithubEveToolModelOutput('getFileContent')).toBe(true) + expect(hasGithubEveToolModelOutput('getRepositoryTree')).toBe(true) expect(hasGithubEveToolModelOutput('listIssues')).toBe(false) expect(formatGithubEveToolOutput('getFileContent', { type: 'file', diff --git a/packages/github-tools/src/eve/registry.ts b/packages/github-tools/src/eve/registry.ts index 51ef801..63e504a 100644 --- a/packages/github-tools/src/eve/registry.ts +++ b/packages/github-tools/src/eve/registry.ts @@ -8,6 +8,7 @@ import { getCommitToModelOutput, getFileContentToModelOutput, getPullRequestContextToModelOutput, + getRepositoryTreeToModelOutput, listPullRequestFilesToModelOutput, } from '../core/model-output' import { stripRateLimit } from '../core/rate-limit' @@ -55,6 +56,7 @@ function modelOutputAdapter( const GITHUB_EVE_TOOL_MODEL_OUTPUT = { getFileContent: modelOutputAdapter(getFileContentToModelOutput), + getRepositoryTree: modelOutputAdapter(getRepositoryTreeToModelOutput), listPullRequestFiles: modelOutputAdapter(listPullRequestFilesToModelOutput), getPullRequestContext: modelOutputAdapter(getPullRequestContextToModelOutput), getCommit: modelOutputAdapter(getCommitToModelOutput), diff --git a/packages/github-tools/src/tools/repository.ts b/packages/github-tools/src/tools/repository.ts index 3c874e6..cedc647 100644 --- a/packages/github-tools/src/tools/repository.ts +++ b/packages/github-tools/src/tools/repository.ts @@ -29,7 +29,7 @@ import { createOrUpdateFileCore, composeCommitMessage, } from '../core/repository' -import { getFileContentToModelOutput } from '../core/model-output' +import { getFileContentToModelOutput, getRepositoryTreeToModelOutput } from '../core/model-output' import { resolveGithubToken, type GithubTokenInput } from '../core/token' import type { CommitToolOptions, ToolOptions, GithubTool } from '../types' @@ -85,6 +85,7 @@ export const getRepositoryTree = (token: GithubTokenInput): GithubTool => tool({ description: getRepositoryTreeDescription, inputSchema: getRepositoryTreeInputSchema, + toModelOutput: getRepositoryTreeToModelOutput, execute: async args => getRepositoryTreeStep({ token: await resolveGithubToken(token), ...args }), })