Skip to content

Commit 389ea2e

Browse files
heiskrCopilot
heiskr
and
Copilot
authored
Add llms.txt generation script, tests, and sync workflow for github.com (#59955)
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 92daa9d commit 389ea2e

4 files changed

Lines changed: 780 additions & 0 deletions

File tree

‎.github/workflows/sync-llms-txt-to-github.yml‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
name: Sync llms.txt to github/github
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: '20 16 * * 1-5' # Weekdays at ~9:20am Pacific
7+
8+
permissions:
9+
contents: read
10+
11+
concurrency:
12+
group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
13+
cancel-in-progress: true
14+
15+
jobs:
16+
sync:
17+
name: Sync llms.txt
18+
if: github.repository == 'github/docs-internal'
19+
runs-on: ubuntu-latest
20+
steps:
21+
- name: Checkout docs-internal
22+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
23+
with:
24+
persist-credentials: false
25+
26+
- uses: ./.github/actions/node-npm-setup
27+
28+
- name: Generate llms.txt
29+
env:
30+
DOCS_BOT_PAT_BASE: ${{ secrets.DOCS_BOT_PAT_BASE }}
31+
run: |
32+
echo "Generating llms.txt from page catalog and popularity data..."
33+
npm run generate-llms-txt --silent > /tmp/llms.txt
34+
echo "Generated llms.txt ($(wc -l < /tmp/llms.txt) lines, $(wc -c < /tmp/llms.txt) bytes)"
35+
36+
- name: Fetch current llms.txt from github/github
37+
id: fetch
38+
env:
39+
GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_BASE }}
40+
run: |
41+
echo "Fetching current public/llms.txt from github/github..."
42+
if gh api repos/github/github/contents/public/llms.txt \
43+
--jq '.content' 2>/dev/null | base64 -d > /tmp/current.txt; then
44+
echo "Fetched current file ($(wc -l < /tmp/current.txt) lines)"
45+
else
46+
echo "No existing file found (first run)"
47+
rm -f /tmp/current.txt
48+
fi
49+
50+
- name: Diff generated vs current
51+
id: diff
52+
run: |
53+
echo "Comparing generated llms.txt against current..."
54+
if [ -f /tmp/current.txt ] && diff -q /tmp/llms.txt /tmp/current.txt > /dev/null 2>&1; then
55+
echo "No changes detected, skipping push"
56+
echo "changed=false" >> "$GITHUB_OUTPUT"
57+
else
58+
echo "Changes detected:"
59+
diff --unified=0 /tmp/current.txt /tmp/llms.txt | head -30 || true
60+
echo "changed=true" >> "$GITHUB_OUTPUT"
61+
fi
62+
63+
- name: Ensure sync branch exists in github/github
64+
if: steps.diff.outputs.changed == 'true'
65+
env:
66+
GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_BASE }}
67+
run: |
68+
BRANCH="auto/sync-llms-txt"
69+
REPO="github/github"
70+
71+
echo "Checking if branch '$BRANCH' exists..."
72+
BRANCH_SHA=$(gh api "repos/$REPO/git/ref/heads/$BRANCH" --jq '.object.sha' 2>/dev/null || true)
73+
if [ -n "$BRANCH_SHA" ]; then
74+
echo "Branch exists at $BRANCH_SHA"
75+
else
76+
echo "Branch does not exist, creating from default branch..."
77+
DEFAULT_BRANCH=$(gh api "repos/$REPO" --jq '.default_branch')
78+
BASE_SHA=$(gh api "repos/$REPO/git/ref/heads/$DEFAULT_BRANCH" --jq '.object.sha')
79+
gh api "repos/$REPO/git/refs" \
80+
--method POST \
81+
-f ref="refs/heads/$BRANCH" \
82+
-f sha="$BASE_SHA"
83+
echo "Created branch from $DEFAULT_BRANCH at $BASE_SHA"
84+
fi
85+
86+
- name: Commit llms.txt to github/github
87+
if: steps.diff.outputs.changed == 'true'
88+
env:
89+
GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_BASE }}
90+
run: |
91+
BRANCH="auto/sync-llms-txt"
92+
REPO="github/github"
93+
CONTENT=$(base64 -w 0 /tmp/llms.txt)
94+
95+
echo "Checking for existing file SHA on branch..."
96+
EXISTING_SHA=$(gh api "repos/$REPO/contents/public/llms.txt?ref=$BRANCH" \
97+
--jq '.sha' 2>/dev/null || true)
98+
if [ -n "$EXISTING_SHA" ]; then
99+
echo "Existing file SHA: $EXISTING_SHA"
100+
else
101+
echo "No existing file on branch (new file)"
102+
fi
103+
104+
echo "Committing llms.txt to $REPO/$BRANCH..."
105+
COMMIT_ARGS=(-f "message=Sync llms.txt from docs.github.com"
106+
-f "content=$CONTENT"
107+
-f "branch=$BRANCH")
108+
if [ -n "$EXISTING_SHA" ]; then
109+
COMMIT_ARGS+=(-f "sha=$EXISTING_SHA")
110+
fi
111+
gh api "repos/$REPO/contents/public/llms.txt" \
112+
--method PUT \
113+
"${COMMIT_ARGS[@]}" --jq '.commit.sha'
114+
echo "Committed successfully"
115+
116+
- name: Create PR if needed
117+
if: steps.diff.outputs.changed == 'true'
118+
env:
119+
GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_BASE }}
120+
run: |
121+
BRANCH="auto/sync-llms-txt"
122+
REPO="github/github"
123+
124+
echo "Checking for existing PR from '$BRANCH'..."
125+
EXISTING_PR=$(gh pr list --repo "$REPO" --head "$BRANCH" \
126+
--json number --jq '.[0].number' 2>/dev/null || true)
127+
if [ -n "$EXISTING_PR" ]; then
128+
echo "PR #$EXISTING_PR already exists, updated with new commit"
129+
exit 0
130+
fi
131+
132+
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
133+
DEFAULT_BRANCH=$(gh api "repos/$REPO" --jq '.default_branch')
134+
135+
PR_BODY="The [sync-llms-txt workflow]($RUN_URL) generated this PR.
136+
137+
Updates \`public/llms.txt\` served at \`github.com/llms.txt\`. The docs-internal script builds this file from the page catalog and popularity data.
138+
139+
No feature flags. Static file in \`public/\`, no code changes.
140+
141+
<!--
142+
Labels for github/github PR template automation:
143+
(\`environment:production-dotcom\`)
144+
(\`risk:low\`)
145+
(\`validate:other\`)
146+
(\`mitigate:rollback\`)
147+
(\`backend/rails/api-only\`)
148+
pull_request_template_version=2
149+
-->"
150+
151+
echo "Creating PR..."
152+
gh pr create \
153+
--repo "$REPO" \
154+
--title "Sync llms.txt from docs.github.com" \
155+
--body "$PR_BODY" \
156+
--head "$BRANCH" \
157+
--base "$DEFAULT_BRANCH" \
158+
--label "docs"
159+
echo "PR created successfully"
160+
161+
- uses: ./.github/actions/slack-alert
162+
if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
163+
with:
164+
slack_channel_id: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
165+
slack_token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
"generate-code-scanning-query-list": "tsx src/codeql-queries/scripts/generate-code-scanning-query-list.ts",
6464
"generate-code-quality-query-list": "tsx src/codeql-queries/scripts/generate-code-quality-query-list.ts",
6565
"generate-content-linter-docs": "tsx src/content-linter/scripts/generate-docs.ts",
66+
"generate-llms-txt": "tsx src/workflows/generate-llms-txt.ts",
6667
"move-content": "tsx src/content-render/scripts/move-content.ts",
6768
"move-by-content-type": "tsx src/content-render/scripts/move-by-content-type.ts",
6869
"openapi-docs": "tsx src/rest/docs.ts",

0 commit comments

Comments
 (0)