Bring in data-directory, let's go async file reads (#16782)

* Bring in data-directory, let's go async file reads

* Lint fixes

* Update glossary.js

lib/data-directory.js

@@ -0,0 +1,68 @@
+const assert = require('assert')
+const fs = require('fs').promises
+const path = require('path')
+const walk = require('walk-sync')
+const yaml = require('js-yaml')
+const { isRegExp, set } = require('lodash')
+const filenameToKey = require('./filename-to-key')
+
+module.exports = async function dataDirectory (dir, opts = {}) {
+  const defaultOpts = {
+    preprocess: (content) => { return content },
+    ignorePatterns: [/README\.md$/i],
+    extensions: [
+      '.json',
+      '.md',
+      '.markdown',
+      '.yaml',
+      '.yml'
+    ]
+  }
+
+  opts = Object.assign({}, defaultOpts, opts)
+
+  // validate input
+  assert(Array.isArray(opts.ignorePatterns))
+  assert(opts.ignorePatterns.every(isRegExp))
+  assert(Array.isArray(opts.extensions))
+  assert(opts.extensions.length)
+
+  // start with an empty data object
+  const data = {}
+
+  // find YAML and Markdown files in the given directory, recursively
+  await Promise.all(walk(dir, { includeBasePath: true })
+    .filter(filename => {
+      // ignore files that match any of ignorePatterns regexes
+      if (opts.ignorePatterns.some(pattern => pattern.test(filename))) return false
+
+      // ignore files that don't have a whitelisted file extension
+      return opts.extensions.includes(path.extname(filename).toLowerCase())
+    })
+    .map(async filename => {
+      // derive `foo.bar.baz` object key from `foo/bar/baz.yml` filename
+      const key = filenameToKey(path.relative(dir, filename))
+      const extension = path.extname(filename).toLowerCase()
+
+      let fileContent = await fs.readFile(filename, 'utf8')
+
+      if (opts.preprocess) fileContent = opts.preprocess(fileContent)
+
+      // add this file's data to the global data object
+      switch (extension) {
+        case '.json':
+          set(data, key, JSON.parse(fileContent))
+          break
+        case '.yaml':
+        case '.yml':
+          set(data, key, yaml.safeLoad(fileContent, { filename }))
+          break
+        case '.md':
+        case '.markdown':
+          set(data, key, fileContent)
+          break
+      }
+    }))
+
+  return data
+}

lib/filename-to-key.js

@@ -0,0 +1,28 @@
+/* eslint-disable prefer-regex-literals */
+const path = require('path')
+const { escapeRegExp } = require('lodash')
+
+// slash at the beginning of a filename
+const leadingPathSeparator = new RegExp(`^${escapeRegExp(path.sep)}`)
+const windowsLeadingPathSeparator = new RegExp('^/')
+
+// all slashes in the filename. path.sep is OS agnostic (windows, mac, etc)
+const pathSeparator = new RegExp(escapeRegExp(path.sep), 'g')
+const windowsPathSeparator = new RegExp('/', 'g')
+
+// handle MS Windows style double-backslashed filenames
+const windowsDoubleSlashSeparator = new RegExp('\\\\', 'g')
+
+// derive `foo.bar.baz` object key from `foo/bar/baz.yml` filename
+module.exports = function filenameToKey (filename) {
+  const extension = new RegExp(`${path.extname(filename)}$`)
+  const key = filename
+    .replace(extension, '')
+    .replace(leadingPathSeparator, '')
+    .replace(windowsLeadingPathSeparator, '')
+    .replace(pathSeparator, '.')
+    .replace(windowsPathSeparator, '.')
+    .replace(windowsDoubleSlashSeparator, '.')
+
+  return key
+}

lib/site-data.js

@@ -2,12 +2,12 @@ const path = require('path')
 const flat = require('flat')
 const { get, set } = require('lodash')
 const languages = require('./languages')
-const dataDirectory = require('@github-docs/data-directory')
+const dataDirectory = require('./data-directory')
 const encodeBracketedParentheticals = require('./encode-bracketed-parentheticals')
 
-const loadSiteDataFromDir = dir => ({
+const loadSiteDataFromDir = async dir => ({
   site: {
-    data: dataDirectory(path.join(dir, 'data'), {
+    data: await dataDirectory(path.join(dir, 'data'), {
       preprocess: dataString =>
         encodeBracketedParentheticals(dataString.trimEnd()),
       ignorePatterns: [/README\.md$/]
@@ -18,7 +18,7 @@ const loadSiteDataFromDir = dir => ({
 module.exports = async function loadSiteData () {
   // load english site data
   const siteData = {
-    en: loadSiteDataFromDir(languages.en.dir)
+    en: await loadSiteDataFromDir(languages.en.dir)
   }
 
   // load and add other language data to siteData where keys match english keys,
@@ -26,7 +26,7 @@ module.exports = async function loadSiteData () {
   const englishKeys = Object.keys(flat(siteData.en))
   for (const language of Object.values(languages)) {
     if (language.code === 'en') continue
-    const data = loadSiteDataFromDir(language.dir)
+    const data = await loadSiteDataFromDir(language.dir)
     for (const key of englishKeys) {
       set(
         siteData,

package-lock.json

@@ -2969,36 +2969,6 @@
         }
       }
     },
-    "@github-docs/data-directory": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/@github-docs/data-directory/-/data-directory-1.2.0.tgz",
-      "integrity": "sha512-hp+Ubwl8e77EdnR4OncSUIE7G/cMn9ENOo6ABy8FjqdYCbAWgb/79w7yXVebIV5P3q5r6KAAqPzHj1N5SSrBgw==",
-      "requires": {
-        "lodash": "^4.17.15",
-        "walk-sync": "^2.0.2"
-      },
-      "dependencies": {
-        "matcher-collection": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/matcher-collection/-/matcher-collection-2.0.1.tgz",
-          "integrity": "sha512-daE62nS2ZQsDg9raM0IlZzLmI2u+7ZapXBwdoeBUKAYERPDDIc0qNqA8E0Rp2D+gspKR7BgIFP52GeujaGXWeQ==",
-          "requires": {
-            "@types/minimatch": "^3.0.3",
-            "minimatch": "^3.0.2"
-          }
-        },
-        "walk-sync": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/walk-sync/-/walk-sync-2.1.0.tgz",
-          "integrity": "sha512-KpH9Xw64LNSx7/UI+3guRZvJWlDxVA4+KKb/4puRoVrG8GkvZRxnF3vhxdjgpoKJGL2TVg1OrtkXIE/VuGPLHQ==",
-          "requires": {
-            "@types/minimatch": "^3.0.3",
-            "ensure-posix-path": "^1.1.0",
-            "matcher-collection": "^2.0.0"
-          }
-        }
-      }
-    },
     "@github-docs/frontmatter": {
       "version": "1.3.1",
       "resolved": "https://registry.npmjs.org/@github-docs/frontmatter/-/frontmatter-1.3.1.tgz",
@@ -5301,7 +5271,7 @@
     },
     "agentkeepalive": {
       "version": "2.2.0",
-      "resolved": "http://registry.npmjs.org/agentkeepalive/-/agentkeepalive-2.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-2.2.0.tgz",
       "integrity": "sha1-xdG9SxKQCPEWPyNvhuX66iAm4u8="
     },
     "aggregate-error": {
@@ -5447,7 +5417,7 @@
     "argparse": {
       "version": "1.0.10",
       "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
-      "integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
       "requires": {
         "sprintf-js": "~1.0.2"
       }
@@ -6824,7 +6794,7 @@
     },
     "brfs": {
       "version": "1.6.1",
-      "resolved": "http://registry.npmjs.org/brfs/-/brfs-1.6.1.tgz",
+      "resolved": "https://registry.npmjs.org/brfs/-/brfs-1.6.1.tgz",
       "integrity": "sha512-OfZpABRQQf+Xsmju8XE9bDjs+uU4vLREGolP7bDgcpsI17QREyZ4Bl+2KLxxx1kCgA0fAIhKQBaBYh+PEcCqYQ==",
       "requires": {
         "quote-stream": "^1.0.1",
@@ -9433,7 +9403,7 @@
     "error-ex": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
-      "integrity": "sha1-tKxAZIEH/c3PriQvQovqihTU8b8=",
+      "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==",
       "requires": {
         "is-arrayish": "^0.2.1"
       }
@@ -12577,7 +12547,7 @@
       "dependencies": {
         "mkdirp": {
           "version": "0.3.0",
-          "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz",
           "integrity": "sha1-G79asbqCevI1dRQ0kEJkVfSB/h4="
         },
         "nopt": {
@@ -17703,7 +17673,7 @@
     },
     "magic-string": {
       "version": "0.22.5",
-      "resolved": "http://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz",
+      "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz",
       "integrity": "sha512-oreip9rJZkzvA8Qzk9HFs8fZGF/u7H/gtrE8EN6RjKJ9kh2HlC+yQ2QezifqTZfGyiuAV0dRv5a+y/8gBb1m9w==",
       "requires": {
         "vlq": "^0.2.2"

package.json

@@ -19,7 +19,6 @@
     "@babel/preset-env": "^7.12.7",
     "@babel/preset-react": "^7.12.7",
     "@babel/runtime": "^7.11.2",
-    "@github-docs/data-directory": "^1.2.0",
     "@github-docs/frontmatter": "^1.3.1",
     "@graphql-inspector/core": "^2.3.0",
     "@graphql-tools/load": "^6.2.5",

tests/unit/data-directory/filename-to-key.js

@@ -0,0 +1,15 @@
+const filenameToKey = require('../../../lib/filename-to-key')
+
+describe('filename-to-key', () => {
+  test('converts filenames to object keys', () => {
+    expect(filenameToKey('foo/bar/baz.txt')).toBe('foo.bar.baz')
+  })
+
+  test('ignores leading slash on filenames', () => {
+    expect(filenameToKey('/foo/bar/baz.txt')).toBe('foo.bar.baz')
+  })
+
+  test('supports MS Windows paths', () => {
+    expect(filenameToKey('path\\to\\file.txt')).toBe('path.to.file')
+  })
+})

tests/unit/data-directory/fixtures/README.md

@@ -0,0 +1 @@
+I am a README. I am ignored by default.

tests/unit/data-directory/fixtures/bar.yaml

@@ -0,0 +1 @@
+another_markup_language: 'yes'

tests/unit/data-directory/fixtures/foo.json

@@ -0,0 +1 @@
+{"meaningOfLife": 42}

tests/unit/data-directory/fixtures/nested/baz.md

@@ -0,0 +1 @@
+I am markdown!

tests/unit/data-directory/index.js

@@ -0,0 +1,40 @@
+const path = require('path')
+const dataDirectory = require('../../../lib/data-directory')
+const fixturesDir = path.join(__dirname, 'fixtures')
+
+describe('data-directory', () => {
+  test('works', async () => {
+    const data = await dataDirectory(fixturesDir)
+    const expected = {
+      bar: { another_markup_language: 'yes' },
+      foo: { meaningOfLife: 42 },
+      nested: { baz: 'I am markdown!' }
+    }
+    expect(data).toEqual(expected)
+  })
+
+  test('option: preprocess function', async () => {
+    const preprocess = function (content) {
+      return content.replace('markdown', 'MARKDOWN')
+    }
+    const data = await dataDirectory(fixturesDir, { preprocess })
+    expect(data.nested.baz).toBe('I am MARKDOWN!')
+  })
+
+  test('option: extensions array', async () => {
+    const extensions = ['.yaml', 'markdown']
+    const data = await dataDirectory(fixturesDir, { extensions })
+    expect('bar' in data).toBe(true)
+    expect('foo' in data).toBe(false) // JSON file should be ignored
+  })
+
+  test('option: ignorePatterns', async () => {
+    const ignorePatterns = []
+
+    // README is ignored by default
+    expect('README' in await dataDirectory(fixturesDir)).toBe(false)
+
+    // README can be included by setting empty ignorePatterns array
+    expect('README' in await dataDirectory(fixturesDir, { ignorePatterns })).toBe(true)
+  })
+})