Remove hubdown and use our own markdown pipeline (#17551)

* Uninstall hubdown, install unified deps

* Create create-processor

* Use it

* Update a doc

* Tweak the language

* Use the same versions that hubdown used

Author: JasonEtco

contributing/content-markup-reference.md

@@ -16,7 +16,7 @@
 
 [Markdown](http://daringfireball.net/projects/markdown/) is a human-friendly syntax for formatting plain text. Our documentation is written with [GitHub Flavored Markdown](https://docs.github.com/en/github/writing-on-github/about-writing-and-formatting-on-github), a custom version of Markdown used across GitHub.
 
-This site's Markdown rendering is powered by the [`/lib/render-content`](/lib/render-content) and [`hubdown`](https://github.com/electron/hubdown) npm packages, which are in turn built on the [`remark`](https://remark.js.org/) Markdown processor.
+This site's Markdown rendering is powered by [`/lib/render-content`](/lib/render-content), which is in turn built on the [`remark`](https://remark.js.org/) Markdown processor.
 
 ## Callout tags
 

lib/render-content/create-processor.js

@@ -0,0 +1,25 @@
+const unified = require('unified')
+const markdown = require('remark-parse')
+const emoji = require('remark-gemoji-to-emoji')
+const remark2rehype = require('remark-rehype')
+const raw = require('rehype-raw')
+const slug = require('rehype-slug')
+const autolinkHeadings = require('rehype-autolink-headings')
+const highlight = require('rehype-highlight')
+const html = require('rehype-stringify')
+const graphql = require('highlightjs-graphql').definer
+const remarkCodeExtra = require('remark-code-extra')
+const codeHeader = require('./plugins/code-header')
+
+module.exports = function createProcessor () {
+  return unified()
+    .use(markdown)
+    .use(remarkCodeExtra, { transform: codeHeader })
+    .use(emoji)
+    .use(remark2rehype, { allowDangerousHTML: true })
+    .use(slug)
+    .use(autolinkHeadings, { behavior: 'wrap' })
+    .use(highlight, { languages: { graphql }, subset: false })
+    .use(raw)
+    .use(html)
+}

lib/render-content/renderContent.js

@@ -1,11 +1,9 @@
 const liquid = require('./liquid')
-const codeHeader = require('./plugins/code-header')
-const hubdown = require('hubdown')
-const remarkCodeExtra = require('remark-code-extra')
 const cheerio = require('cheerio')
 const Entities = require('html-entities').XmlEntities
 const entities = new Entities()
 const stripHtmlComments = require('strip-html-comments')
+const createProcessor = require('./create-processor')
 
 // used below to remove extra newlines in TOC lists
 const endLine = '</a>\r?\n'
@@ -54,13 +52,9 @@ module.exports = async function renderContent (
     // statements so that extra space doesn't mess with list numbering
     template = template.replace(/(\r?\n){3}/g, '\n\n')
 
-    let { content: html } = await hubdown(template, {
-      // Disable automatic language guessing in syntax highlighting
-      highlight: { subset: false },
-      runBefore: [[
-        remarkCodeExtra, { transform: codeHeader }
-      ]]
-    })
+    const processor = createProcessor()
+    const vFile = await processor.process(template)
+    let html = vFile.toString()
 
     // Remove unwanted newlines (which appear as spaces) from inline tags inside tables
     if (html.includes('<table>')) html = removeNewlinesFromInlineTags(html)

package-lock.json

@@ -52,10 +52,10 @@
     "hast-util-from-parse5": "^6.0.1",
     "hastscript": "^6.0.0",
     "helmet": "^3.21.2",
+    "highlightjs-graphql": "^1.0.2",
     "hot-shots": "^8.2.0",
     "html-entities": "^1.2.1",
     "html-truncate": "^1.2.2",
-    "hubdown": "^2.6.0",
     "imurmurhash": "^0.1.4",
     "ioredis": "^4.19.4",
     "ioredis-mock": "^5.2.0",
@@ -79,7 +79,15 @@
     "react": "^17.0.1",
     "react-dom": "^17.0.1",
     "readline-sync": "^1.4.10",
+    "rehype-autolink-headings": "^2.0.5",
+    "rehype-highlight": "^3.1.0",
+    "rehype-raw": "^4.0.2",
+    "rehype-slug": "^2.0.3",
+    "rehype-stringify": "^6.0.1",
     "remark-code-extra": "^1.0.1",
+    "remark-gemoji-to-emoji": "^1.1.0",
+    "remark-parse": "^7.0.2",
+    "remark-rehype": "^5.0.0",
     "resolve-url-loader": "^3.1.2",
     "rimraf": "^3.0.0",
     "sass": "^1.26.3",
@@ -89,6 +97,7 @@
     "slash": "^3.0.0",
     "strip-html-comments": "^1.0.0",
     "style-loader": "^1.2.1",
+    "unified": "^8.4.2",
     "unist-util-visit": "^2.0.3",
     "uuid": "^8.3.0",
     "walk-sync": "^1.1.4",