Skip to content

Commit 02f2090

Browse files
authored
feat(registry): add Skill context checks beside token usage (#530)
1 parent 20b0517 commit 02f2090

4 files changed

Lines changed: 176 additions & 1 deletion

File tree

‎layers/registry/app/components/SkillDetail.vue‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import { avatarProxyUrl, githubAvatarProxyUrl } from '#shared/image-proxy'
1414
import { skillPageUrl as exactSkillPageUrl, skillInstallCmd, skillRunCmd, skillRunPrompt } from '#shared/skill-commands'
1515
import { headlineTrendingAward, trendingAwardBadgeLabel, trendingAwardLabel, trendingAwardPath } from '#shared/trending-award'
1616
import { behaviorIcon } from '../utils/skill-behaviors'
17+
import { resolveSkillContextChecks } from '../utils/skill-context-checks'
1718
import { formatByteSize, formatTokenCount, resolveSkillContextCost, resolveSkillFileContext } from '../utils/skill-context-cost'
1819
import { fileIcon, highlightLangFromPath } from '../utils/skill-file-tree'
1920
import { partitionMetadataEntries } from '../utils/skill-metadata'
@@ -539,6 +540,11 @@ const contextCost = computed(() => data.value
539540
})
540541
: null)
541542
543+
const contextChecks = computed(() => resolveSkillContextChecks({
544+
raw: data.value?.raw ?? null,
545+
frontmatter: data.value?.frontmatter ?? null,
546+
}))
547+
542548
const audits = computed<SkillAudit[]>(() => liveSkill.value?.audits ?? [])
543549
const auditOverview = computed(() => resolveSkillAuditOverview(audits.value))
544550
@@ -2164,7 +2170,7 @@ useHead(computed(() => ({
21642170
aria-hidden="true"
21652171
/>
21662172
<span v-if="viewerContext._tag === 'skill'">
2167-
<strong>{{ formatTokenCount(viewerContext.cost.tokens.metadata) }}</strong> tokens always: the name and description.
2173+
<strong>{{ formatTokenCount(viewerContext.cost.tokens.metadata) }}</strong> tokens for metadata: the name and description.
21682174
<strong>{{ formatTokenCount(viewerContext.cost.tokens.instructions) }}</strong> when used: this file.
21692175
<template v-if="viewerContext.cost.resourceFileCount">
21702176
<strong>{{ formatTokenCount(viewerContext.cost.tokens.resources) }}</strong> more on demand in {{ viewerContext.cost.resourceFileCount }} {{ viewerContext.cost.resourceFileCount === 1 ? 'file' : 'files' }}.
@@ -2180,6 +2186,10 @@ useHead(computed(() => ({
21802186
Your agent does not load this file into context unless it opens it.
21812187
</span>
21822188
</p>
2189+
<SkillContextChecks
2190+
v-if="viewerContext?._tag === 'skill' && data?.raw"
2191+
:result="contextChecks"
2192+
/>
21832193
<section
21842194
v-show="contentView === 'preview'"
21852195
class="skill-mdxg p-4 sm:p-6 relative"
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
<script setup lang="ts">
2+
import type { resolveSkillContextChecks } from '../utils/skill-context-checks'
3+
4+
const { result } = defineProps<{
5+
result: ReturnType<typeof resolveSkillContextChecks>
6+
}>()
7+
8+
const hasError = computed(() => result.checks.some(check => check.tone === 'error'))
9+
const hasDescriptionCap = computed(() => result.checks.some(check => check.code === 'codex-description-cap' || check.code === 'claude-description-cap'))
10+
</script>
11+
12+
<template>
13+
<details class="group border-b border-default font-mono text-xs">
14+
<summary
15+
class="flex cursor-pointer list-none items-center gap-2 px-4 py-2 focus-visible:outline-2 focus-visible:outline-primary sm:px-6 [&::-webkit-details-marker]:hidden"
16+
:class="hasError ? 'text-error-700 dark:text-error' : result.checks.length ? 'text-warning-800 dark:text-warning' : 'text-muted'"
17+
>
18+
<UIcon
19+
:name="result.checks.length ? 'i-lucide-triangle-alert' : 'i-lucide-info'"
20+
class="size-3.5 shrink-0"
21+
aria-hidden="true"
22+
/>
23+
<span v-if="hasError">SKILL.md has missing or invalid fields</span>
24+
<span v-else-if="hasDescriptionCap">An Agent shortens this description</span>
25+
<span v-else-if="result.checks.length && result.claudeListing._tag === 'listed'">Description uses {{ result.claudeListing.percent.toFixed(1) }}% of example budget</span>
26+
<span v-else>How this Skill uses context</span>
27+
<UIcon name="i-lucide-chevron-down" class="ml-auto size-3.5 shrink-0 group-open:rotate-180" aria-hidden="true" />
28+
</summary>
29+
<div class="space-y-3 px-4 pb-3 font-sans text-sm sm:px-6">
30+
<p v-if="!hasError" class="text-default">
31+
Before choosing a Skill, your Agent reads its name and description.
32+
All available Skills share that space.
33+
</p>
34+
<ul v-if="result.checks.length" class="space-y-2" aria-label="Skill context checks">
35+
<li v-for="check in result.checks" :key="check.code" :class="check.tone === 'error' ? 'text-error-700 dark:text-error' : 'text-warning-800 dark:text-warning'">
36+
{{ check.message }}
37+
</li>
38+
</ul>
39+
<div v-if="result.claudeListing._tag === 'listed'" class="space-y-2 text-muted">
40+
<p>
41+
In our Claude Code example, all Skill names and descriptions share 8,000 characters.
42+
This Skill uses ≈{{ result.claudeListing.characters }} characters, or {{ result.claudeListing.percent.toFixed(1) }}%.
43+
</p>
44+
<p v-if="result.checks.some(check => check.code === 'listing-share')">
45+
The 1% threshold is a size suggestion. Longer descriptions can still fit.
46+
</p>
47+
<p>Your model, settings, and other Skills decide how much text your Agent can read.</p>
48+
</div>
49+
<p v-else-if="result.claudeListing._tag === 'explicit-only'" class="text-muted">
50+
This Skill turns off automatic use in Claude Code.
51+
Its description stays out of the automatic Skill list by default.
52+
</p>
53+
<details class="text-muted">
54+
<summary class="cursor-pointer text-xs font-mono">
55+
Example settings and source
56+
</summary>
57+
<p class="mt-2">
58+
The example uses a 200k-token context and default Claude Code settings.
59+
The count includes the name, description, separators, and when_to_use when present.
60+
Codex also counts local file paths.
61+
</p>
62+
<p class="mt-2 text-xs">
63+
<a href="https://github.com/smol-ai/skit/blob/08ff55d9d995b402ebd29c51bdedbe5690b9b0e0/docs/skill-context-budgets.md" target="_blank" rel="noopener noreferrer" class="underline underline-offset-2">Skit's source and limits</a>:
64+
Codex 0.160.1, Claude Code 2.1.292.
65+
</p>
66+
</details>
67+
</div>
68+
</details>
69+
</template>
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/** Source-only checks. No local Agent configuration or complete listing is available here. */
2+
export interface SkillContextCheck {
3+
code: 'frontmatter' | 'name' | 'description' | 'codex-description-cap' | 'claude-description-cap' | 'listing-share'
4+
tone: 'warning' | 'error'
5+
message: string
6+
}
7+
8+
type ClaudeListing
9+
= | { _tag: 'unavailable' }
10+
| { _tag: 'explicit-only' }
11+
| { _tag: 'listed', characters: number, percent: number }
12+
13+
// Versioned observations, not tokenizer counts or a visitor's configured allowance.
14+
// https://github.com/smol-ai/skit/blob/08ff55d9d995b402ebd29c51bdedbe5690b9b0e0/docs/skill-context-budgets.md
15+
// Codex 0.160.1: 1,024 code points; Claude Code 2.1.292: 1,536 combined code points.
16+
const CODEX_DESCRIPTION_CAP = 1024
17+
const CLAUDE_DESCRIPTION_CAP = 1536
18+
const CLAUDE_EXAMPLE_ALLOWANCE = 8000 // 200,000 context tokens × 4 characters × 1%.
19+
20+
export function resolveSkillContextChecks(input: {
21+
raw: string | null
22+
frontmatter: Record<string, unknown> | null
23+
}): { checks: SkillContextCheck[], claudeListing: ClaudeListing } {
24+
const checks: SkillContextCheck[] = []
25+
if (!input.raw)
26+
return { checks, claudeListing: { _tag: 'unavailable' } }
27+
const fields = input.frontmatter
28+
if (!fields) {
29+
checks.push({ code: 'frontmatter', tone: 'error', message: 'SKILL.md needs a valid YAML header with a name and description.' })
30+
return { checks, claudeListing: { _tag: 'unavailable' } }
31+
}
32+
const name = typeof fields.name === 'string' ? fields.name.trim() : ''
33+
const description = typeof fields.description === 'string' ? fields.description.trim() : ''
34+
if (!name)
35+
checks.push({ code: 'name', tone: 'error', message: 'The name in the SKILL.md header is missing or is not text.' })
36+
if (!description)
37+
checks.push({ code: 'description', tone: 'error', message: 'The description in the SKILL.md header is missing or is not text.' })
38+
if (!name || !description)
39+
return { checks, claudeListing: { _tag: 'unavailable' } }
40+
41+
if (Array.from(description).length > CODEX_DESCRIPTION_CAP)
42+
checks.push({ code: 'codex-description-cap', tone: 'warning', message: 'Codex shortens descriptions longer than 1,024 characters. Put when-to-use details first.' })
43+
44+
const explicitOnly = fields['disable-model-invocation'] === true || fields['disable-model-invocation'] === 'true'
45+
if (explicitOnly)
46+
return { checks, claudeListing: { _tag: 'explicit-only' } }
47+
48+
const combined = [description, typeof fields.when_to_use === 'string' ? fields.when_to_use : ''].filter(Boolean).join(' ').trim()
49+
const combinedCharacters = Array.from(combined)
50+
if (combinedCharacters.length > CLAUDE_DESCRIPTION_CAP)
51+
checks.push({ code: 'claude-description-cap', tone: 'warning', message: 'Claude Code shortens description plus when_to_use after 1,536 characters by default. Put when-to-use details first.' })
52+
53+
// Matches Skit's estimated name, separator, capped description, and newline row.
54+
const characters = Array.from(`${name}: ${combinedCharacters.slice(0, CLAUDE_DESCRIPTION_CAP).join('')}\n`).length
55+
const percent = characters / CLAUDE_EXAMPLE_ALLOWANCE * 100
56+
if (characters * 100 > CLAUDE_EXAMPLE_ALLOWANCE)
57+
checks.push({ code: 'listing-share', tone: 'warning', message: 'A shorter description leaves more room for other Skills. This entry exceeds our 1% size suggestion.' })
58+
return { checks, claudeListing: { _tag: 'listed', characters, percent } }
59+
}
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import { describe, expect, it } from 'vitest'
2+
import { resolveSkillContextChecks } from '../../layers/registry/app/utils/skill-context-checks'
3+
4+
const source = (frontmatter: Record<string, unknown> | null) => ({ raw: 'source', frontmatter })
5+
6+
describe('skill context checks', () => {
7+
it('counts the rendered Claude row and flags strictly above 1% of the example allowance', () => {
8+
const atLimit = resolveSkillContextChecks(source({ name: 'x', description: 'a'.repeat(76) }))
9+
const above = resolveSkillContextChecks(source({ name: 'x', description: 'a'.repeat(77) }))
10+
expect(atLimit.claudeListing).toEqual({ _tag: 'listed', characters: 80, percent: 1 })
11+
expect(atLimit.checks).toEqual([])
12+
expect(above.checks.map(check => check.code)).toEqual(['listing-share'])
13+
})
14+
15+
it('caps Unicode descriptions by characters and includes when_to_use for Claude', () => {
16+
const result = resolveSkillContextChecks(source({ name: 'x', description: '😀'.repeat(1025), when_to_use: 'b'.repeat(512) }))
17+
expect(result.checks.map(check => check.code)).toEqual(['codex-description-cap', 'claude-description-cap', 'listing-share'])
18+
expect(result.claudeListing).toEqual({ _tag: 'listed', characters: 1540, percent: 19.25 })
19+
})
20+
21+
it('keeps explicit-only Claude Skills out of the listing estimate', () => {
22+
const result = resolveSkillContextChecks(source({ 'name': 'x', 'description': 'a'.repeat(2000), 'disable-model-invocation': true }))
23+
expect(result.claudeListing).toEqual({ _tag: 'explicit-only' })
24+
expect(result.checks.map(check => check.code)).toEqual(['codex-description-cap'])
25+
})
26+
27+
it('reports missing or invalid source metadata without counting repository fallback text', () => {
28+
expect(resolveSkillContextChecks(source(null)).checks.map(check => check.code)).toEqual(['frontmatter'])
29+
const invalid = resolveSkillContextChecks(source({ name: 5, description: ['not a string'] }))
30+
expect(invalid.checks.map(check => check.code)).toEqual(['name', 'description'])
31+
expect(invalid.claudeListing).toEqual({ _tag: 'unavailable' })
32+
})
33+
34+
it('does not report source defects while the source is unavailable', () => {
35+
expect(resolveSkillContextChecks({ raw: null, frontmatter: null })).toEqual({ checks: [], claudeListing: { _tag: 'unavailable' } })
36+
})
37+
})

0 commit comments

Comments
 (0)