Merge pull request #18483 from github/script-to-move-toc-links-into-frontmatter

Script to move TOC links into children frontmatter

Author: sarahs

lib/get-document-type.js

@@ -0,0 +1,17 @@
+module.exports = function getDocumentType (relativePath) {
+  if (!relativePath.endsWith('index.md')) {
+    return 'article'
+  }
+
+  // Derive the document type from the path segment length
+  switch (relativePath.split('/').length) {
+    case 1:
+      return 'homepage'
+    case 2:
+      return 'product'
+    case 3:
+      return 'category'
+    case 4:
+      return 'mapTopic'
+  }
+}

script/content-migrations/remove-map-topics.js

@@ -3,23 +3,31 @@
 const fs = require('fs')
 const path = require('path')
 const walk = require('walk-sync')
+const languages = require('../../lib/languages')
+const frontmatter = require('../../lib/read-frontmatter')
+const addRedirectToFrontmatter = require('../../lib/redirects/add-redirect-to-frontmatter')
 
 const relativeRefRegex = /\/[a-zA-Z0-9-]+/g
 const linkString = /{% [^}]*?link.*? \/(.*?) ?%}/m
 const linksArray = new RegExp(linkString.source, 'gm')
 
-const fullDirectoryPath = path.join(process.cwd(), '/content')
-const files = walk(fullDirectoryPath, {
+const walkOpts = {
   includeBasePath: true,
   directories: false
-})
+}
+
+// We only want category TOC files, not product TOCs.
+const categoryFileRegex = /content\/[^/]+?\/[^/]+?\/index.md/
 
-files.forEach(file => {
-  if (path.basename(file) !== 'index.md') return
+const fullDirectoryPaths = Object.values(languages).map(langObj => path.join(process.cwd(), langObj.dir, 'content'))
+const categoryIndexFiles = fullDirectoryPaths.map(fullDirectoryPath => walk(fullDirectoryPath, walkOpts)).flat()
+  .filter(file => categoryFileRegex.test(file))
+
+categoryIndexFiles.forEach(categoryIndexFile => {
+  let categoryIndexContent = fs.readFileSync(categoryIndexFile, 'utf8')
 
-  let fileContent = fs.readFileSync(file, 'utf-8')
   // find array of TOC link strings
-  const rawItems = fileContent.match(linksArray)
+  const rawItems = categoryIndexContent.match(linksArray)
   if (!rawItems || !rawItems[0].includes('topic_link_in_list')) return
 
   const pageToc = {}
@@ -37,24 +45,53 @@ files.forEach(file => {
       pageToc[currentTopic] = tmpArray
     }
   })
+
   for (const topic in pageToc) {
-    const oldTopicDirectory = path.dirname(file)
+    const oldTopicDirectory = path.dirname(categoryIndexFile)
     const newTopicDirectory = path.join(oldTopicDirectory, topic)
-    const topicFile = path.join(oldTopicDirectory, `${topic}.md`)
+    const oldTopicFile = path.join(oldTopicDirectory, `${topic}.md`)
+
+    // Some translated category TOCs may be outdated and contain incorrect links
+    if (!fs.existsSync(oldTopicFile)) continue
 
     if (!fs.existsSync(newTopicDirectory)) fs.mkdirSync(newTopicDirectory)
 
-    let topicContent = fs.readFileSync(topicFile, 'utf-8')
-    topicContent = topicContent.replace('mapTopic: true\n', '')
+    const { data, content } = frontmatter(fs.readFileSync(oldTopicFile, 'utf8'))
+    delete data.mapTopic
+
+    let topicContent = content
 
     const articles = pageToc[topic]
 
     articles.forEach(article => {
-      fs.renameSync(`${oldTopicDirectory}/${article}.md`, `${newTopicDirectory}/${article}.md`)
+      // Update the new map topic index file content
       topicContent = topicContent + `{% link_with_intro /${article} %}\n`
-      fileContent = fileContent.replace(`/{% link_in_list /${article}`, `/{% link_in_list /${newTopicDirectory}/${article}`)
+
+      // Update the category index file content
+      categoryIndexContent = categoryIndexContent.replace(`{% link_in_list /${article}`, `{% link_in_list /${topic}/${article}`)
+
+      // Early return if the article doesn't exist (some translated category TOCs may be outdated and contain incorrect links)
+      if (!fs.existsSync(`${oldTopicDirectory}/${article}.md`)) return
+
+      // Move the file under the new map topic directory
+      const newArticlePath = `${newTopicDirectory}/${article}.md`
+      fs.renameSync(`${oldTopicDirectory}/${article}.md`, newArticlePath)
+
+      // Read the article file so we can add a redirect from its old path
+      const articleContents = frontmatter(fs.readFileSync(newArticlePath, 'utf8'))
+      addRedirectToFrontmatter(articleContents.data.redirect_from, `${oldTopicDirectory}/${article}`)
+
+      // Write the article with updated frontmatter
+      fs.writeFileSync(newArticlePath, frontmatter.stringify(articleContents.content.trim(), articleContents.data, { lineWidth: 10000 }))
     })
-    fs.writeFileSync(`${newTopicDirectory}/index.md`, topicContent)
-    fs.unlinkSync(topicFile)
+
+    // Write the map topic index file
+    fs.writeFileSync(`${newTopicDirectory}/index.md`, frontmatter.stringify(topicContent.trim(), data, { lineWidth: 10000 }))
+
+    // Write the category index file
+    fs.writeFileSync(categoryIndexFile, categoryIndexContent)
+
+    // Delete the old map topic
+    fs.unlinkSync(oldTopicFile)
   }
 })

script/content-migrations/update-tocs.js

@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+
+const fs = require('fs')
+const path = require('path')
+const walk = require('walk-sync')
+const yaml = require('js-yaml')
+const frontmatter = require('../../lib/read-frontmatter')
+const getDocumentType = require('../../lib/get-document-type')
+const languages = require('../../lib/languages')
+
+const linkString = /{% [^}]*?link.*? (\/.*?) ?%}/m
+const linksArray = new RegExp(linkString.source, 'gm')
+
+// The product order is determined by data/products.yml
+const productsFile = path.join(process.cwd(), 'data/products.yml')
+const productsYml = yaml.load(fs.readFileSync(productsFile, 'utf8'))
+const sortedProductIds = productsYml.productsInOrder.concat('early-access')
+
+// This script turns `{% link /<link> %} style content into children: [ -/<link> ] frontmatter arrays.
+//
+// It MUST be run after script/content-migrations/remove-map-topics.js.
+//
+// NOTE: The results won't work with the TOC handling currently in production, so the results must NOT
+// be committed until the updated handling is in place.
+
+const walkOpts = {
+  includeBasePath: true,
+  directories: false
+}
+
+const fullDirectoryPaths = Object.values(languages).map(langObj => path.join(process.cwd(), langObj.dir, 'content'))
+const indexFiles = fullDirectoryPaths.map(fullDirectoryPath => walk(fullDirectoryPath, walkOpts)).flat()
+  .filter(file => file.endsWith('index.md'))
+
+indexFiles
+  .forEach(indexFile => {
+    const relativePath = indexFile.replace(/^.+\/content\//, '')
+    const documentType = getDocumentType(relativePath)
+
+    const { data, content } = frontmatter(fs.readFileSync(indexFile, 'utf8'))
+
+    if (documentType === 'homepage') {
+      data.children = sortedProductIds
+    }
+
+    const linkItems = content.match(linksArray) || []
+
+    // Turn the `{% link /<link> %}` list into an array of /<link> items
+    if (documentType === 'product' || documentType === 'mapTopic') {
+      data.children = getLinks(linkItems)
+    }
+
+    if (documentType === 'category') {
+      const childMapTopics = linkItems.filter(item => item.includes('topic_'))
+
+      data.children = childMapTopics.length ? getLinks(childMapTopics) : getLinks(linkItems)
+    }
+
+    // Fix this one weird file
+    if (relativePath === 'discussions/guides/index.md') {
+      data.children = [
+        '/best-practices-for-community-conversations-on-github',
+        '/finding-discussions-across-multiple-repositories',
+        '/granting-higher-permissions-to-top-contributors'
+      ]
+    }
+
+    // Index files should no longer have body content, so we write an empty string
+    fs.writeFileSync(indexFile, frontmatter.stringify('', data, { lineWidth: 10000 }))
+  })
+
+function getLinks (linkItemArray) {
+  // do a oneoff replacement while mapping
+  return linkItemArray.map(item => item.match(linkString)[1].replace('/discussions-guides', '/guides'))
+}