Move data-directory to also mapLimit (#16921)

Co-authored-by: Chiedo John <2156688+chiedo@users.noreply.github.com>

Author: heiskr

lib/data-directory.js

@@ -2,10 +2,13 @@ const assert = require('assert')
 const fs = require('fs').promises
 const path = require('path')
 const walk = require('walk-sync')
+const { mapLimit } = require('async')
 const yaml = require('js-yaml')
 const { isRegExp, set } = require('lodash')
 const filenameToKey = require('./filename-to-key')
 
+const FILE_READ_LIMIT = 100
+
 module.exports = async function dataDirectory (dir, opts = {}) {
   const defaultOpts = {
     preprocess: (content) => { return content },
@@ -31,15 +34,18 @@ module.exports = async function dataDirectory (dir, opts = {}) {
   const data = {}
 
   // find YAML and Markdown files in the given directory, recursively
-  await Promise.all(walk(dir, { includeBasePath: true })
+  const filenames = walk(dir, { includeBasePath: true })
     .filter(filename => {
       // ignore files that match any of ignorePatterns regexes
       if (opts.ignorePatterns.some(pattern => pattern.test(filename))) return false
 
       // ignore files that don't have a whitelisted file extension
       return opts.extensions.includes(path.extname(filename).toLowerCase())
     })
-    .map(async filename => {
+  await mapLimit(
+    filenames,
+    FILE_READ_LIMIT,
+    async filename => {
       // derive `foo.bar.baz` object key from `foo/bar/baz.yml` filename
       const key = filenameToKey(path.relative(dir, filename))
       const extension = path.extname(filename).toLowerCase()
@@ -62,7 +68,8 @@ module.exports = async function dataDirectory (dir, opts = {}) {
           set(data, key, fileContent)
           break
       }
-    }))
+    }
+  )
 
   return data
 }

lib/pages.js

@@ -3,7 +3,8 @@ const walk = require('walk-sync').entries
 const Page = require('./page')
 const languages = require('./languages')
 const { mapLimit, filterLimit } = require('async')
-const FILE_READ_LIMIT = 500
+
+const FILE_READ_LIMIT = 100
 
 async function loadPageList () {
   const pageList = []