Support Accept: text/markdown content negotiation and extend .md to all page types (#59999)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: heiskr

src/frame/middleware/cache-control.ts

@@ -84,20 +84,26 @@ export function defaultCacheControl(res: Response): void {
   defaultBrowserCacheControl(res)
 }
 
+// Vary on content type for pages that support content negotiation (HTML vs markdown)
+export function contentTypeCacheControl(res: Response): void {
+  defaultCacheControl(res)
+  res.append('vary', 'accept')
+}
+
 // Vary on language when needed
 // x-user-language is a custom request header derived from req.cookie:user_language
 // accept-language is truncated to one of our available languages
 // https://bit.ly/3u5UeRN
 export function languageCacheControl(res: Response): void {
   defaultCacheControl(res)
-  res.set('vary', 'accept-language, x-user-language')
+  res.append('vary', 'accept-language, x-user-language')
 }
 
 // Vary on both language and version for homepage redirects
 // x-user-version is a custom request header derived from req.cookie:user_version
 export function languageAndVersionCacheControl(res: Response): void {
   defaultCacheControl(res)
-  res.set('vary', 'accept-language, x-user-language, x-user-version')
+  res.append('vary', 'accept-language, x-user-language, x-user-version')
 }
 
 // Long cache control for versioned assets: images, CSS, JS...

src/frame/middleware/render-page.ts

@@ -10,7 +10,7 @@ import statsd from '@/observability/lib/statsd'
 import type { ExtendedRequest } from '@/types'
 import { allVersions } from '@/versions/lib/all-versions'
 import { minimumNotFoundHtml } from '../lib/constants'
-import { defaultCacheControl } from './cache-control'
+import { contentTypeCacheControl, defaultCacheControl } from './cache-control'
 import { isConnectionDropped } from './halt-on-dropped-connection'
 import { nextHandleRequest } from './next'
 
@@ -90,6 +90,12 @@ export default async function renderPage(req: ExtendedRequest, res: Response) {
   // Stop processing if the connection was already dropped
   if (isConnectionDropped(req, res)) return
 
+  // Content negotiation: serve markdown when the client prefers it over HTML.
+  // Agents like Claude Code send Accept headers that omit text/html.
+  if (req.accepts(['text/html', 'text/markdown']) === 'text/markdown') {
+    context.markdownRequested = true
+  }
+
   if (!req.context) throw new Error('request not contextualized')
   req.context.renderedPage = await buildRenderedPage(req)
   req.context.miniTocItems = buildMiniTocItems(req)
@@ -145,15 +151,11 @@ export default async function renderPage(req: ExtendedRequest, res: Response) {
   }
 
   if (context.markdownRequested) {
-    if (!page.autogenerated && page.documentType === 'article') {
-      return res.type('text/markdown').send(req.context.renderedPage)
-    } else {
-      const newUrl = req.originalUrl.replace(req.path, req.path.replace(/\.md$/, ''))
-      return res.redirect(newUrl)
-    }
+    contentTypeCacheControl(res)
+    return res.type('text/markdown').send(req.context.renderedPage)
   }
 
-  defaultCacheControl(res)
+  contentTypeCacheControl(res)
 
   return nextHandleRequest(req, res)
 }

src/frame/tests/server.ts

@@ -278,6 +278,40 @@ describe('server', () => {
       expect(res.headers['cache-control']).toMatch(/max-age=\d+/)
     })
   })
+
+  describe('Accept: text/markdown content negotiation', () => {
+    test('returns markdown when Accept header prefers text/markdown', async () => {
+      const res = await get('/en', {
+        headers: {
+          accept: 'text/markdown',
+        },
+      })
+      expect(res.statusCode).toBe(200)
+      expect(res.headers['content-type']).toContain('text/markdown')
+      expect(res.headers.vary).toContain('accept')
+    })
+
+    test('returns HTML when Accept header prefers text/html', async () => {
+      const res = await get('/en', {
+        headers: {
+          accept: 'text/html,application/xhtml+xml',
+        },
+      })
+      expect(res.statusCode).toBe(200)
+      expect(res.headers['content-type']).toContain('text/html')
+      expect(res.headers.vary).toContain('accept')
+    })
+
+    test('returns HTML when Accept header is */*', async () => {
+      const res = await get('/en', {
+        headers: {
+          accept: '*/*',
+        },
+      })
+      expect(res.statusCode).toBe(200)
+      expect(res.headers['content-type']).toContain('text/html')
+    })
+  })
 })
 
 describe('static routes', () => {

src/shielding/middleware/handle-invalid-paths.ts

@@ -85,21 +85,12 @@ export default function handleInvalidPaths(
 
   if (req.path.endsWith('/index.md')) {
     defaultCacheControl(res)
-    // The originalUrl is the full URL including query string.
-    // E.g. `/en/foo.md?bar=baz`
     const newUrl = req.originalUrl.replace(req.path, req.path.replace(/\/index\.md$/, ''))
     return res.redirect(newUrl)
   } else if (req.path.endsWith('.md')) {
-    // encode the query params but also make them pretty so we can see
-    // them as `/` and `@` in the address bar
-    // e.g. /api/article/body?pathname=/en/enterprise-server@3.16/admin...
-    // NOT: /api/article/body?pathname=%2Fen%2Fenterprise-server%403.16%2Fadmin...
-    const encodedPath = encodeURIComponent(req.path.replace(/\.md$/, ''))
-      .replace(/%2F/g, '/')
-      .replace(/%40/g, '@')
-    const newUrl = `/api/article/body?pathname=${encodedPath}`
-    res.redirect(newUrl)
-    return
+    req.url = req.url.replace(/\.md($|\?)/, '$1')
+    req.headers.accept = 'text/markdown'
+    return next()
   }
   return next()
 }

src/shielding/tests/shielding.ts

@@ -72,24 +72,12 @@ describe('index.md and .md suffixes', () => {
     }
   })
 
-  test('any URL that ends with /.md redirects', async () => {
-    // With language prefix
-    {
-      const res = await get('/en/get-started/hello.md')
-      expect(res.statusCode).toBe(302)
-      expect(res.headers.location).toBe('/api/article/body?pathname=/en/get-started/hello')
-    }
-    // Without language prefix
+  test('any URL that ends with .md serves markdown directly', async () => {
+    // .md is stripped and request flows through with Accept: text/markdown
     {
-      const res = await get('/get-started/hello.md')
-      expect(res.statusCode).toBe(302)
-      expect(res.headers.location).toBe('/api/article/body?pathname=/get-started/hello')
-    }
-    // With query string
-    {
-      const res = await get('/get-started/hello.md?foo=bar')
-      expect(res.statusCode).toBe(302)
-      expect(res.headers.location).toBe('/api/article/body?pathname=/get-started/hello')
+      const res = await get('/en/get-started.md')
+      // Should not redirect — serves markdown directly (or 404 if page doesn't exist)
+      expect(res.statusCode).not.toBe(302)
     }
   })
 })