repo sync

Author: Octomerger

lib/data-directory.js

@@ -2,10 +2,13 @@ const assert = require('assert')
 const fs = require('fs').promises
 const path = require('path')
 const walk = require('walk-sync')
+const { mapLimit } = require('async')
 const yaml = require('js-yaml')
 const { isRegExp, set } = require('lodash')
 const filenameToKey = require('./filename-to-key')
 
+const FILE_READ_LIMIT = 100
+
 module.exports = async function dataDirectory (dir, opts = {}) {
   const defaultOpts = {
     preprocess: (content) => { return content },
@@ -31,15 +34,18 @@ module.exports = async function dataDirectory (dir, opts = {}) {
   const data = {}
 
   // find YAML and Markdown files in the given directory, recursively
-  await Promise.all(walk(dir, { includeBasePath: true })
+  const filenames = walk(dir, { includeBasePath: true })
     .filter(filename => {
       // ignore files that match any of ignorePatterns regexes
       if (opts.ignorePatterns.some(pattern => pattern.test(filename))) return false
 
       // ignore files that don't have a whitelisted file extension
       return opts.extensions.includes(path.extname(filename).toLowerCase())
     })
-    .map(async filename => {
+  await mapLimit(
+    filenames,
+    FILE_READ_LIMIT,
+    async filename => {
       // derive `foo.bar.baz` object key from `foo/bar/baz.yml` filename
       const key = filenameToKey(path.relative(dir, filename))
       const extension = path.extname(filename).toLowerCase()
@@ -62,7 +68,8 @@ module.exports = async function dataDirectory (dir, opts = {}) {
           set(data, key, fileContent)
           break
       }
-    }))
+    }
+  )
 
   return data
 }

lib/pages.js

@@ -3,7 +3,8 @@ const walk = require('walk-sync').entries
 const Page = require('./page')
 const languages = require('./languages')
 const { mapLimit, filterLimit } = require('async')
-const FILE_READ_LIMIT = 500
+
+const FILE_READ_LIMIT = 100
 
 async function loadPageList () {
   const pageList = []

lib/path-utils.js

@@ -6,6 +6,8 @@ const allProducts = require('./all-products')
 const allVersions = require('./all-versions')
 const { getNewVersionedPath } = require('./old-versions-utils')
 
+const supportedVersions = new Set(Object.keys(allVersions))
+
 // construct appropriate versioned path for any given HREF
 function getVersionedPathWithoutLanguage (href, version) {
   // start clean without language code or trailing slash
@@ -24,7 +26,7 @@ function getVersionedPathWithoutLanguage (href, version) {
 
   // if the version found is not a currently supported version...
   let productObjectFromPath
-  if (!Object.keys(allVersions).includes(versionFromPath)) {
+  if (!supportedVersions.has(versionFromPath)) {
     // first check if the first segment is instead a current product;
     // example: /admin/foo or /desktop/foo
     productObjectFromPath = allProducts[versionFromPath]