Article API: Add title/intro consistency and refactor transformers (#59103)

Author: heiskr

src/article-api/lib/graphql-helpers.ts

@@ -0,0 +1,32 @@
+import type { Context, Page } from '@/types'
+import { renderContent } from '@/content-render/index'
+import matter from '@gr2m/gray-matter'
+
+/**
+ * Extract manual content from page markdown
+ * Used by GraphQL transformers to get content before the auto-generated marker
+ */
+export async function extractManualContent(page: Page, context: Context): Promise<string> {
+  if (!page.markdown) return ''
+
+  const markerIndex = page.markdown.indexOf(
+    '<!-- Content after this section is automatically generated -->',
+  )
+
+  if (markerIndex <= 0) return ''
+
+  const { content } = matter(page.markdown)
+  const manualContentMarkerIndex = content.indexOf(
+    '<!-- Content after this section is automatically generated -->',
+  )
+
+  if (manualContentMarkerIndex <= 0) return ''
+
+  const rawManualContent = content.substring(0, manualContentMarkerIndex).trim()
+  if (!rawManualContent) return ''
+
+  return await renderContent(rawManualContent, {
+    ...context,
+    markdownRequested: true,
+  })
+}

src/article-api/middleware/article-body.ts

@@ -74,5 +74,20 @@ export async function getArticleBody(req: ExtendedRequestWithPageInfo) {
   // these parts allow us to render the page
   const renderingReq = await createContextualizedRenderingRequest(pathname, page)
   renderingReq.context.markdownRequested = true
-  return await page.render(renderingReq.context)
+  const content = await page.render(renderingReq.context)
+
+  // Get title and intro for consistency with transformer-based pages
+  const title = page.title
+  const intro = page.intro
+    ? await page.renderProp('intro', renderingReq.context, { textOnly: true })
+    : ''
+
+  // Prepend title and intro to the content
+  let result = `# ${title}\n\n`
+  if (intro) {
+    result += `${intro}\n\n`
+  }
+  result += content
+
+  return result
 }

src/article-api/templates/codeql-cli-page.template.md

@@ -0,0 +1,7 @@
+# {{ page.title }}
+
+{% if page.intro %}
+{{ page.intro }}
+{% endif %}
+
+{{ content }}

src/article-api/templates/secret-scanning-page.template.md

@@ -0,0 +1,7 @@
+# {{ page.title }}
+
+{% if page.intro %}
+{{ page.intro }}
+{% endif %}
+
+{{ content }}

src/article-api/templates/webhooks-page.template.md

@@ -11,12 +11,42 @@
 {% for webhook in webhooks %}
 ## {{ webhook.name }}
 
-**Available actions:** {% for actionType in webhook.actionTypes %}{% if forloop.last and forloop.length > 1 %}and {% endif %}`{{ actionType }}`{% unless forloop.last %}{% if forloop.length > 2 %}, {% else %} {% endif %}{% endunless %}{% endfor %}
+{% if webhook.summary %}
+{{ webhook.summary }}
 
-{% if webhook.data.descriptionHtml %}
-{{ webhook.data.descriptionHtml }}
 {% endif %}
+{% if webhook.availability.size > 0 %}
+### Availability
 
-**Availability:** {% for availability in webhook.data.availability %}{% if forloop.last and forloop.length > 1 %}and {% endif %}`{{ availability }}`{% unless forloop.last %}{% if forloop.length > 2 %}, {% else %} {% endif %}{% endunless %}{% endfor %}
+{% for avail in webhook.availability %}- `{{ avail }}`
+{% endfor %}
+
+{% endif %}
+### Webhook payload object
+
+{% if webhook.actionTypes.size > 1 %}
+**Action type:** {% for actionType in webhook.actionTypes %}`{{ actionType }}`{% unless forloop.last %}, {% endunless %}{% endfor %}
+
+{% endif %}
+{% if webhook.description %}
+{{ webhook.description }}
+
+{% endif %}
+{% if webhook.bodyParameters.size > 0 %}
+#### Webhook payload object parameters
+
+| Name | Type | Description |
+|------|------|-------------|
+{% for param in webhook.bodyParameters %}| `{{ param.name }}` | `{{ param.type }}` | {% if param.isRequired %}**Required.** {% endif %}{{ param.description }} |
+{% endfor %}
 
+{% endif %}
+{% if webhook.payloadExample %}
+### Webhook payload example
+
+```json
+{{ webhook.payloadExample }}
+```
+
+{% endif %}
 {% endfor %}

src/article-api/tests/article-body.ts

@@ -25,6 +25,15 @@ describe('article body api', () => {
     expect(res.headers['content-type']).toContain('text/markdown')
   })
 
+  test('body includes title and intro', async () => {
+    const res = await get(makeURL('/en/get-started/start-your-journey/hello-world'))
+    expect(res.statusCode).toBe(200)
+    // Body should start with the page title as H1
+    expect(res.body).toMatch(/^# Hello World/)
+    // Body should include the intro after the title
+    expect(res.body).toContain('Follow this Hello World exercise to get started with')
+  })
+
   test('octicons auto-generate aria-labels', async () => {
     const res = await get(makeURL('/en/get-started/start-your-journey/hello-world'))
     expect(res.statusCode).toBe(200)

src/article-api/tests/webhooks-transformer.ts

@@ -93,8 +93,8 @@ describe('Webhooks transformer', () => {
     // Should show payload object parameters section
     expect(res.body).toContain('### Webhook payload object')
     expect(res.body).toContain('#### Webhook payload object parameters')
-    // Should have a markdown table with parameter columns
-    expect(res.body).toContain('| Name | Type | Description |')
+    // Should have a markdown table with parameter columns (may have extra spacing from formatting)
+    expect(res.body).toMatch(/\|\s*Name\s*\|\s*Type\s*\|\s*Description\s*\|/)
   })
 
   test('webhooks show descriptions', async () => {

src/article-api/transformers/audit-logs-transformer.ts

@@ -2,19 +2,16 @@ import type { Context, Page } from '@/types'
 import type { PageTransformer } from './types'
 import type { CategorizedEvents } from '@/audit-logs/types'
 import { renderContent } from '@/content-render/index'
+import { loadTemplate } from '@/article-api/lib/load-template'
 import matter from '@gr2m/gray-matter'
-import { readFileSync } from 'fs'
-import { join, dirname } from 'path'
-import { fileURLToPath } from 'url'
-
-const __filename = fileURLToPath(import.meta.url)
-const __dirname = dirname(__filename)
 
 /**
  * Transformer for Audit Logs pages
  * Converts audit log events and their data into markdown format using a Liquid template
  */
 export class AuditLogsTransformer implements PageTransformer {
+  templateName = 'audit-logs-page.template.md'
+
   canTransform(page: Page): boolean {
     return page.autogenerated === 'audit-logs'
   }
@@ -76,8 +73,7 @@ export class AuditLogsTransformer implements PageTransformer {
     )
 
     // Load and render template
-    const templatePath = join(__dirname, '../templates/audit-logs-page.template.md')
-    const templateContent = readFileSync(templatePath, 'utf8')
+    const templateContent = loadTemplate(this.templateName)
 
     // Render the template with Liquid
     const rendered = await renderContent(templateContent, {

src/article-api/transformers/codeql-cli-transformer.ts

@@ -1,28 +1,47 @@
 import type { Context, Page } from '@/types'
 import type { PageTransformer } from './types'
+import { renderContent } from '@/content-render/index'
+import { loadTemplate } from '@/article-api/lib/load-template'
 import { stripHtmlCommentsAndNormalizeWhitespace } from '@/article-api/lib/strip-html-comments'
 
 /**
  * Transformer for CodeQL CLI reference pages.
- * Renders autogenerated CodeQL CLI documentation pages as markdown.
+ * Renders autogenerated CodeQL CLI documentation pages as markdown using a Liquid template.
  * Sets `markdownRequested` to true in the context to ensure the page is rendered as markdown,
  * bypassing the default article type check.
  */
 export class CodeQLCliTransformer implements PageTransformer {
+  templateName = 'codeql-cli-page.template.md'
+
   canTransform(page: Page): boolean {
     return page.autogenerated === 'codeql-cli'
   }
 
   async transform(page: Page, _pathname: string, context: Context): Promise<string> {
     // CodeQL CLI pages are fully generated markdown files in the repo.
-    // We render them with markdownRequested=true to get the markdown output,
-    // similar to how regular articles are rendered but through the transformer pattern.
+    // We render them with markdownRequested=true to get the markdown output.
     context.markdownRequested = true
     const content = await page.render(context)
 
     const intro = page.intro ? await page.renderProp('intro', context, { textOnly: true }) : ''
 
-    const result = `# ${page.title}\n\n${intro}\n\n${content}`
+    // Prepare template data
+    const templateData: Record<string, unknown> = {
+      page: {
+        title: page.title,
+        intro,
+      },
+      content,
+    }
+
+    // Load and render template
+    const templateContent = loadTemplate(this.templateName)
+
+    const result = await renderContent(templateContent, {
+      ...context,
+      ...templateData,
+      markdownRequested: true,
+    })
 
     // Strip HTML comments (e.g., markdownlint-disable comments) from the output
     return stripHtmlCommentsAndNormalizeWhitespace(result)

src/article-api/transformers/github-apps-transformer.ts

@@ -1,13 +1,9 @@
 import type { Context, Page } from '@/types'
 import type { PageTransformer } from './types'
 import { renderContent } from '@/content-render/index'
+import { loadTemplate } from '@/article-api/lib/load-template'
 import matter from '@gr2m/gray-matter'
-import { readFileSync } from 'fs'
-import { join, dirname } from 'path'
-import { fileURLToPath } from 'url'
 
-const __filename = fileURLToPath(import.meta.url)
-const __dirname = dirname(__filename)
 const DEBUG = process.env.RUNNER_DEBUG === '1' || process.env.DEBUG === '1'
 
 // GitHub Apps data types
@@ -91,6 +87,8 @@ const PERMISSIONS_PAGE_TYPES = new Set([
  * in TypeScript for permissions pages to avoid Liquid escaping issues.
  */
 export class GithubAppsTransformer implements PageTransformer {
+  templateName = 'github-apps-page.template.md'
+
   canTransform(page: Page): boolean {
     return page.autogenerated === 'github-apps'
   }
@@ -160,9 +158,8 @@ export class GithubAppsTransformer implements PageTransformer {
       isPermissionsPage,
     )
 
-    // Load and render template
-    const templatePath = join(__dirname, '../templates/github-apps-page.template.md')
-    const templateContent = readFileSync(templatePath, 'utf8')
+    // Load template
+    const templateContent = loadTemplate(this.templateName)
 
     // For permissions pages, we need to construct the tables manually to avoid Liquid escaping
     let finalContent: string