diff --git a/lib/data-directory.js b/lib/data-directory.js new file mode 100644 index 0000000000..d72abcbf55 --- /dev/null +++ b/lib/data-directory.js @@ -0,0 +1,68 @@ +const assert = require('assert') +const fs = require('fs').promises +const path = require('path') +const walk = require('walk-sync') +const yaml = require('js-yaml') +const { isRegExp, set } = require('lodash') +const filenameToKey = require('./filename-to-key') + +module.exports = async function dataDirectory (dir, opts = {}) { + const defaultOpts = { + preprocess: (content) => { return content }, + ignorePatterns: [/README\.md$/i], + extensions: [ + '.json', + '.md', + '.markdown', + '.yaml', + '.yml' + ] + } + + opts = Object.assign({}, defaultOpts, opts) + + // validate input + assert(Array.isArray(opts.ignorePatterns)) + assert(opts.ignorePatterns.every(isRegExp)) + assert(Array.isArray(opts.extensions)) + assert(opts.extensions.length) + + // start with an empty data object + const data = {} + + // find YAML and Markdown files in the given directory, recursively + await Promise.all(walk(dir, { includeBasePath: true }) + .filter(filename => { + // ignore files that match any of ignorePatterns regexes + if (opts.ignorePatterns.some(pattern => pattern.test(filename))) return false + + // ignore files that don't have a whitelisted file extension + return opts.extensions.includes(path.extname(filename).toLowerCase()) + }) + .map(async filename => { + // derive `foo.bar.baz` object key from `foo/bar/baz.yml` filename + const key = filenameToKey(path.relative(dir, filename)) + const extension = path.extname(filename).toLowerCase() + + let fileContent = await fs.readFile(filename, 'utf8') + + if (opts.preprocess) fileContent = opts.preprocess(fileContent) + + // add this file's data to the global data object + switch (extension) { + case '.json': + set(data, key, JSON.parse(fileContent)) + break + case '.yaml': + case '.yml': + set(data, key, yaml.safeLoad(fileContent, { filename })) + break + case '.md': + case '.markdown': + set(data, key, fileContent) + break + } + })) + + return data +} diff --git a/lib/filename-to-key.js b/lib/filename-to-key.js new file mode 100644 index 0000000000..568552285b --- /dev/null +++ b/lib/filename-to-key.js @@ -0,0 +1,28 @@ +/* eslint-disable prefer-regex-literals */ +const path = require('path') +const { escapeRegExp } = require('lodash') + +// slash at the beginning of a filename +const leadingPathSeparator = new RegExp(`^${escapeRegExp(path.sep)}`) +const windowsLeadingPathSeparator = new RegExp('^/') + +// all slashes in the filename. path.sep is OS agnostic (windows, mac, etc) +const pathSeparator = new RegExp(escapeRegExp(path.sep), 'g') +const windowsPathSeparator = new RegExp('/', 'g') + +// handle MS Windows style double-backslashed filenames +const windowsDoubleSlashSeparator = new RegExp('\\\\', 'g') + +// derive `foo.bar.baz` object key from `foo/bar/baz.yml` filename +module.exports = function filenameToKey (filename) { + const extension = new RegExp(`${path.extname(filename)}$`) + const key = filename + .replace(extension, '') + .replace(leadingPathSeparator, '') + .replace(windowsLeadingPathSeparator, '') + .replace(pathSeparator, '.') + .replace(windowsPathSeparator, '.') + .replace(windowsDoubleSlashSeparator, '.') + + return key +} diff --git a/lib/site-data.js b/lib/site-data.js index 7c4ba0c988..bdd47f1ce7 100755 --- a/lib/site-data.js +++ b/lib/site-data.js @@ -2,12 +2,12 @@ const path = require('path') const flat = require('flat') const { get, set } = require('lodash') const languages = require('./languages') -const dataDirectory = require('@github-docs/data-directory') +const dataDirectory = require('./data-directory') const encodeBracketedParentheticals = require('./encode-bracketed-parentheticals') -const loadSiteDataFromDir = dir => ({ +const loadSiteDataFromDir = async dir => ({ site: { - data: dataDirectory(path.join(dir, 'data'), { + data: await dataDirectory(path.join(dir, 'data'), { preprocess: dataString => encodeBracketedParentheticals(dataString.trimEnd()), ignorePatterns: [/README\.md$/] @@ -18,7 +18,7 @@ const loadSiteDataFromDir = dir => ({ module.exports = async function loadSiteData () { // load english site data const siteData = { - en: loadSiteDataFromDir(languages.en.dir) + en: await loadSiteDataFromDir(languages.en.dir) } // load and add other language data to siteData where keys match english keys, @@ -26,7 +26,7 @@ module.exports = async function loadSiteData () { const englishKeys = Object.keys(flat(siteData.en)) for (const language of Object.values(languages)) { if (language.code === 'en') continue - const data = loadSiteDataFromDir(language.dir) + const data = await loadSiteDataFromDir(language.dir) for (const key of englishKeys) { set( siteData, diff --git a/package-lock.json b/package-lock.json index b0f2a9c3e7..e4cee8054d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2969,36 +2969,6 @@ } } }, - "@github-docs/data-directory": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@github-docs/data-directory/-/data-directory-1.2.0.tgz", - "integrity": "sha512-hp+Ubwl8e77EdnR4OncSUIE7G/cMn9ENOo6ABy8FjqdYCbAWgb/79w7yXVebIV5P3q5r6KAAqPzHj1N5SSrBgw==", - "requires": { - "lodash": "^4.17.15", - "walk-sync": "^2.0.2" - }, - "dependencies": { - "matcher-collection": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/matcher-collection/-/matcher-collection-2.0.1.tgz", - "integrity": "sha512-daE62nS2ZQsDg9raM0IlZzLmI2u+7ZapXBwdoeBUKAYERPDDIc0qNqA8E0Rp2D+gspKR7BgIFP52GeujaGXWeQ==", - "requires": { - "@types/minimatch": "^3.0.3", - "minimatch": "^3.0.2" - } - }, - "walk-sync": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/walk-sync/-/walk-sync-2.1.0.tgz", - "integrity": "sha512-KpH9Xw64LNSx7/UI+3guRZvJWlDxVA4+KKb/4puRoVrG8GkvZRxnF3vhxdjgpoKJGL2TVg1OrtkXIE/VuGPLHQ==", - "requires": { - "@types/minimatch": "^3.0.3", - "ensure-posix-path": "^1.1.0", - "matcher-collection": "^2.0.0" - } - } - } - }, "@github-docs/frontmatter": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/@github-docs/frontmatter/-/frontmatter-1.3.1.tgz", @@ -5301,7 +5271,7 @@ }, "agentkeepalive": { "version": "2.2.0", - "resolved": "http://registry.npmjs.org/agentkeepalive/-/agentkeepalive-2.2.0.tgz", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-2.2.0.tgz", "integrity": "sha1-xdG9SxKQCPEWPyNvhuX66iAm4u8=" }, "aggregate-error": { @@ -5447,7 +5417,7 @@ "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", "requires": { "sprintf-js": "~1.0.2" } @@ -6824,7 +6794,7 @@ }, "brfs": { "version": "1.6.1", - "resolved": "http://registry.npmjs.org/brfs/-/brfs-1.6.1.tgz", + "resolved": "https://registry.npmjs.org/brfs/-/brfs-1.6.1.tgz", "integrity": "sha512-OfZpABRQQf+Xsmju8XE9bDjs+uU4vLREGolP7bDgcpsI17QREyZ4Bl+2KLxxx1kCgA0fAIhKQBaBYh+PEcCqYQ==", "requires": { "quote-stream": "^1.0.1", @@ -9433,7 +9403,7 @@ "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", - "integrity": "sha1-tKxAZIEH/c3PriQvQovqihTU8b8=", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", "requires": { "is-arrayish": "^0.2.1" } @@ -12577,7 +12547,7 @@ "dependencies": { "mkdirp": { "version": "0.3.0", - "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz", "integrity": "sha1-G79asbqCevI1dRQ0kEJkVfSB/h4=" }, "nopt": { @@ -17703,7 +17673,7 @@ }, "magic-string": { "version": "0.22.5", - "resolved": "http://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz", "integrity": "sha512-oreip9rJZkzvA8Qzk9HFs8fZGF/u7H/gtrE8EN6RjKJ9kh2HlC+yQ2QezifqTZfGyiuAV0dRv5a+y/8gBb1m9w==", "requires": { "vlq": "^0.2.2" diff --git a/package.json b/package.json index a0cae4097a..545de58826 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,6 @@ "@babel/preset-env": "^7.12.7", "@babel/preset-react": "^7.12.7", "@babel/runtime": "^7.11.2", - "@github-docs/data-directory": "^1.2.0", "@github-docs/frontmatter": "^1.3.1", "@graphql-inspector/core": "^2.3.0", "@graphql-tools/load": "^6.2.5", diff --git a/tests/unit/data-directory/filename-to-key.js b/tests/unit/data-directory/filename-to-key.js new file mode 100644 index 0000000000..052f78cf98 --- /dev/null +++ b/tests/unit/data-directory/filename-to-key.js @@ -0,0 +1,15 @@ +const filenameToKey = require('../../../lib/filename-to-key') + +describe('filename-to-key', () => { + test('converts filenames to object keys', () => { + expect(filenameToKey('foo/bar/baz.txt')).toBe('foo.bar.baz') + }) + + test('ignores leading slash on filenames', () => { + expect(filenameToKey('/foo/bar/baz.txt')).toBe('foo.bar.baz') + }) + + test('supports MS Windows paths', () => { + expect(filenameToKey('path\\to\\file.txt')).toBe('path.to.file') + }) +}) diff --git a/tests/unit/data-directory/fixtures/README.md b/tests/unit/data-directory/fixtures/README.md new file mode 100644 index 0000000000..fb62e0bc60 --- /dev/null +++ b/tests/unit/data-directory/fixtures/README.md @@ -0,0 +1 @@ +I am a README. I am ignored by default. \ No newline at end of file diff --git a/tests/unit/data-directory/fixtures/bar.yaml b/tests/unit/data-directory/fixtures/bar.yaml new file mode 100644 index 0000000000..d028f54fed --- /dev/null +++ b/tests/unit/data-directory/fixtures/bar.yaml @@ -0,0 +1 @@ +another_markup_language: 'yes' diff --git a/tests/unit/data-directory/fixtures/foo.json b/tests/unit/data-directory/fixtures/foo.json new file mode 100644 index 0000000000..8fd3eb5c42 --- /dev/null +++ b/tests/unit/data-directory/fixtures/foo.json @@ -0,0 +1 @@ +{"meaningOfLife": 42} \ No newline at end of file diff --git a/tests/unit/data-directory/fixtures/nested/baz.md b/tests/unit/data-directory/fixtures/nested/baz.md new file mode 100644 index 0000000000..c2be33651c --- /dev/null +++ b/tests/unit/data-directory/fixtures/nested/baz.md @@ -0,0 +1 @@ +I am markdown! \ No newline at end of file diff --git a/tests/unit/data-directory/index.js b/tests/unit/data-directory/index.js new file mode 100644 index 0000000000..0a6bdcf793 --- /dev/null +++ b/tests/unit/data-directory/index.js @@ -0,0 +1,40 @@ +const path = require('path') +const dataDirectory = require('../../../lib/data-directory') +const fixturesDir = path.join(__dirname, 'fixtures') + +describe('data-directory', () => { + test('works', async () => { + const data = await dataDirectory(fixturesDir) + const expected = { + bar: { another_markup_language: 'yes' }, + foo: { meaningOfLife: 42 }, + nested: { baz: 'I am markdown!' } + } + expect(data).toEqual(expected) + }) + + test('option: preprocess function', async () => { + const preprocess = function (content) { + return content.replace('markdown', 'MARKDOWN') + } + const data = await dataDirectory(fixturesDir, { preprocess }) + expect(data.nested.baz).toBe('I am MARKDOWN!') + }) + + test('option: extensions array', async () => { + const extensions = ['.yaml', 'markdown'] + const data = await dataDirectory(fixturesDir, { extensions }) + expect('bar' in data).toBe(true) + expect('foo' in data).toBe(false) // JSON file should be ignored + }) + + test('option: ignorePatterns', async () => { + const ignorePatterns = [] + + // README is ignored by default + expect('README' in await dataDirectory(fixturesDir)).toBe(false) + + // README can be included by setting empty ignorePatterns array + expect('README' in await dataDirectory(fixturesDir, { ignorePatterns })).toBe(true) + }) +})