JIT data (#32140)

components/context/MainContext.tsx

@@ -40,7 +40,7 @@ type DataT = {
       version_was_deprecated: string
       version_will_be_deprecated: string
       deprecation_details: string
-      isOldestReleaseDeprecated: boolean
+      isOldestReleaseDeprecated?: boolean
     }
     policies: {
       translation: string
@@ -130,12 +130,27 @@ export const getMainContext = async (req: any, res: any): Promise<MainContextT>
     error: req.context.error ? req.context.error.toString() : '',
     data: {
       ui: req.context.site.data.ui,
+
       reusables: {
-        enterprise_deprecation: req.context.site.data.reusables.enterprise_deprecation,
+        enterprise_deprecation: {
-        policies: req.context.site.data.reusables.policies,
+          version_was_deprecated: req.context.getDottedData(
+            'reusables.enterprise_deprecation.version_was_deprecated'
+          ),
+          version_will_be_deprecated: req.context.getDottedData(
+            'reusables.enterprise_deprecation.version_will_be_deprecated'
+          ),
+          deprecation_details: req.context.getDottedData(
+            'reusables.enterprise_deprecation.deprecation_details'
+          ),
+        },
+        policies: {
+          translation: req.context.getDottedData('reusables.policies.translation'),
+        },
       },
       variables: {
-        release_candidate: req.context.site.data.variables.release_candidate,
+        release_candidate: {
+          version: req.context.getDottedData('variables.release_candidate.version') || null,
+        },
       },
     },
     currentCategory: req.context.currentCategory || '',

content/get-started/quickstart/github-glossary.md

@@ -13,7 +13,7 @@ versions:
 ---
 {% for glossary in glossaries %}
   ### {{ glossary.term }}
-  {{ glossary.description}}
+  {{ glossary.description }}
   ---
 {% endfor %}
 

lib/get-applicable-versions.js

@@ -1,12 +1,8 @@
-import path from 'path'
 import { reduce, sortBy } from 'lodash-es'
 import { allVersions } from './all-versions.js'
 import versionSatisfiesRange from './version-satisfies-range.js'
 import checkIfNextVersionOnly from './check-if-next-version-only.js'
-import dataDirectory from './data-directory.js'
+import { getDeepDataByLanguage } from './get-data.js'
-import encodeBracketedParentheses from './encode-bracketed-parentheses.js'
-
-const featuresDir = path.join('data', 'features')
 
 let featureData = null
 
@@ -24,10 +20,7 @@ function getApplicableVersions(frontmatterVersions, filepath) {
   }
 
   if (!featureData) {
-    featureData = dataDirectory(featuresDir, {
+    featureData = getDeepDataByLanguage('features', 'en')
-      preprocess: (dataString) => encodeBracketedParentheses(dataString.trimEnd()),
-      ignorePatterns: [/README\.md$/],
-    })
   }
 
   // Check for frontmatter that includes a feature name, like:

lib/get-data.js

@@ -7,6 +7,10 @@ import { merge, get } from 'lodash-es'
 
 import languages from './languages.js'
 
+// If you run `export DEBUG_JIT_DATA_READS=true` in your terminal,
+// next time it will mention every file it reads from disk.
+const DEBUG_JIT_DATA_READS = Boolean(JSON.parse(process.env.DEBUG_JIT_DATA_READS || 'false'))
+
 // This is a list of files that we should always immediately fall back to
 // English for.
 // Having this is safer than trying to wrangle the translations to NOT
@@ -237,7 +241,7 @@ const getMarkdownContent = memoize((root, relPath, englishRoot) => {
 
 const getFileContent = (root, relPath, englishRoot) => {
   const filePath = root ? path.join(root, relPath) : relPath
-  if (process.env.NODE_ENV === 'development') console.log('READ', filePath)
+  if (DEBUG_JIT_DATA_READS) console.log('READ', filePath)
   try {
     return fs.readFileSync(filePath, 'utf-8')
   } catch (err) {

lib/get-english-headings.js

@@ -1,7 +1,9 @@
 import { fromMarkdown } from 'mdast-util-from-markdown'
 import { toString } from 'mdast-util-to-string'
 import { visit } from 'unist-util-visit'
+
 import findPage from './find-page.js'
+import { getDataByLanguage } from './get-data.js'
 
 // for any translated page, first get corresponding English markdown
 // then get the headings on both the translated and English pageMap
@@ -11,7 +13,7 @@ export default function getEnglishHeadings(page, context) {
   // generated programatically.
   if (page.relativePath.endsWith('/github-glossary.md')) {
     // Return an object of `{ localized-term: english-slug }`
-    const languageGlossary = context.site.data.glossaries.external
+    const languageGlossary = getDataByLanguage('glossaries.external', 'en')
     return languageGlossary.reduce((prev, curr) => {
       prev[curr.term] = curr.slug
       return prev

lib/liquid-tags/data.js

@@ -1,7 +1,7 @@
 import { TokenizationError } from 'liquidjs'
-import matter from 'gray-matter'
 
 import { THROW_ON_EMPTY, DataReferenceError } from './error-handling.js'
+import { getDataByLanguage } from '../get-data.js'
 
 const Syntax = /([a-z0-9/\\_.\-[\]]+)/i
 const SyntaxHelp = "Syntax Error in 'data' - Valid syntax: data [path]"
@@ -16,8 +16,8 @@ export default {
   },
 
   async render(scope) {
-    let value = await this.liquid.evalValue(`site.data.${this.path}`, scope)
+    const text = getDataByLanguage(this.path, scope.environments.currentLanguage)
-    if (typeof value !== 'string') {
+    if (text === undefined) {
       const message = `Can't find the key 'site.data.${this.path}' in the scope.`
       if (THROW_ON_EMPTY) {
         throw new DataReferenceError(message)
@@ -26,12 +26,6 @@ export default {
       return
     }
 
-    // Drop any frontmatter since reusable Markdown files aren't currently
+    return this.liquid.parseAndRender(text, scope.environments)
-    // expected to use frontmatter.  This is useful since our current translation
-    // process adds YAML frontmatter properties to any Markdown file that gets
-    // translated and we don't want to render that information.
-    ;({ content: value } = matter(value))
-
-    return this.liquid.parseAndRender(value, scope.environments)
   },
 }

lib/liquid-tags/indented-data-reference.js

@@ -1,6 +1,7 @@
 import assert from 'assert'
 
 import { THROW_ON_EMPTY, IndentedDataReferenceError } from './error-handling.js'
+import { getDataByLanguage } from '../get-data.js'
 
 // This class supports a tag that expects two parameters, a data reference and `spaces=NUMBER`:
 //
@@ -33,20 +34,8 @@ export default {
     assert(parseInt(numSpaces) || numSpaces === '0', '"spaces=NUMBER" must include a number')
 
     // Get the referenced value from the context
-    const value = await this.liquid.evalValue(`site.data.${dataReference}`, scope)
+    const text = getDataByLanguage(dataReference, scope.environments.currentLanguage)
-
+    if (text === undefined) {
-    // If value is falsy it can be because we completely failed to look
-    // it up. But it can also be literally an empty string.
-    // For example, the reusable could be entirely something like this:
-    //
-    //   {% if some condition %}The meat{% endif %}
-    //
-    // Then it's working as expected. But if the reference is wrong, e.g.
-    //
-    //   {% indented_data_reference reusables.foo.tyypu spaces=3 %}
-    //
-    // Or if the file simple doesn't exist, you get an undefined for the value.
-    if (typeof value !== 'string' && !value) {
       const message = `Can't find the key 'site.data.${dataReference}' in the scope.`
       if (THROW_ON_EMPTY) {
         throw new IndentedDataReferenceError(message)
@@ -56,7 +45,7 @@ export default {
     }
 
     // add spaces to each line
-    const renderedReferenceWithIndent = value.replace(/^/gm, ' '.repeat(numSpaces))
+    const renderedReferenceWithIndent = text.replace(/^/gm, ' '.repeat(numSpaces))
 
     return this.liquid.parseAndRender(renderedReferenceWithIndent, scope.environments)
   },

lib/page-data.js

@@ -1,8 +1,8 @@
 import path from 'path'
+
 import languages from './languages.js'
 import { allVersions } from './all-versions.js'
 import createTree, { getBasePath } from './create-tree.js'
-import loadSiteData from './site-data.js'
 import nonEnterpriseDefaultVersion from './non-enterprise-default-version.js'
 import Page from './page.js'
 
@@ -60,8 +60,7 @@ export async function loadUnversionedTree(languagesOnly = null) {
  *
  * Order of languages and versions doesn't matter, but order of child page arrays DOES matter (for navigation).
 */
-export async function loadSiteTree(unversionedTree, siteData) {
+export async function loadSiteTree(unversionedTree) {
-  const site = siteData || loadSiteData()
   const rawTree = Object.assign({}, unversionedTree || (await loadUnversionedTree()))
   const siteTree = {}
 
@@ -76,8 +75,7 @@ export async function loadSiteTree(unversionedTree, siteData) {
           treePerVersion[version] = await versionPages(
             Object.assign({}, rawTree[langCode]),
             version,
-            langCode,
+            langCode
-            site
           )
         })
       )
@@ -89,7 +87,7 @@ export async function loadSiteTree(unversionedTree, siteData) {
   return siteTree
 }
 
-export async function versionPages(obj, version, langCode, site) {
+export async function versionPages(obj, version, langCode) {
   // Add a versioned href as a convenience for use in layouts.
   obj.href = obj.page.permalinks.find(
     (pl) =>
@@ -103,7 +101,7 @@ export async function versionPages(obj, version, langCode, site) {
       // Drop child pages that do not apply to the current version
       .filter((childPage) => childPage.page.applicableVersions.includes(version))
       // Version the child pages recursively.
-      .map((childPage) => versionPages(Object.assign({}, childPage), version, langCode, site))
+      .map((childPage) => versionPages(Object.assign({}, childPage), version, langCode))
   )
 
   obj.childPages = [...versionedChildPages]

lib/process-learning-tracks.js

@@ -1,6 +1,7 @@
 import renderContent from './render-content/index.js'
 import getLinkData from './get-link-data.js'
 import getApplicableVersions from './get-applicable-versions.js'
+import { getDataByLanguage } from './get-data.js'
 
 const renderOpts = { textOnly: true, encodeEntities: true }
 
@@ -23,11 +24,20 @@ export default async function processLearningTracks(rawLearningTracks, context)
     if (!renderedTrackName) continue
 
     // Find the data for the current product and track name.
-    const trackDataForProduct = context.site.data['learning-tracks'][context.currentProduct]
+
-    if (!trackDataForProduct) {
+    if (context.currentProduct.includes('.')) {
-      throw new Error(`No learning track data for product "${context.currentProduct}".`)
+      throw new Error(`currentProduct can not contain a . (${context.currentProduct})`)
     }
-    const track = trackDataForProduct[renderedTrackName]
+    if (renderedTrackName.includes('.')) {
+      throw new Error(`renderedTrackName can not contain a . (${renderedTrackName})`)
+    }
+
+    // Note: this will use the translated learning tracks and automatically
+    // fall back to English if they don't exist on disk in the translation.
+    const track = getDataByLanguage(
+      `learning-tracks.${context.currentProduct}.${renderedTrackName}`,
+      context.currentLanguage
+    )
     if (!track) {
       throw new Error(`No learning track called '${renderedTrackName}'.`)
     }

lib/warm-server.js

@@ -1,7 +1,6 @@
 import statsd from './statsd.js'
 import { loadUnversionedTree, loadSiteTree, loadPages, loadPageMap } from './page-data.js'
 import loadRedirects from './redirects/precompile.js'
-import loadSiteData from './site-data.js'
 
 // Instrument these functions so that
 // it's wrapped in a timer that reports to Datadog
@@ -11,7 +10,6 @@ const dog = {
   loadPages: statsd.asyncTimer(loadPages, 'load_pages'),
   loadPageMap: statsd.asyncTimer(loadPageMap, 'load_page_map'),
   loadRedirects: statsd.asyncTimer(loadRedirects, 'load_redirects'),
-  loadSiteData: statsd.timer(loadSiteData, 'load_site_data'),
 }
 
 // For multiple-triggered Promise sharing
@@ -25,8 +23,7 @@ async function warmServer() {
   }
 
   const unversionedTree = await dog.loadUnversionedTree()
-  const site = dog.loadSiteData()
+  const siteTree = await dog.loadSiteTree(unversionedTree)
-  const siteTree = await dog.loadSiteTree(unversionedTree, site)
   const pageList = await dog.loadPages(unversionedTree)
   const pageMap = await dog.loadPageMap(pageList)
   const redirects = await dog.loadRedirects(pageList)
@@ -43,7 +40,6 @@ async function warmServer() {
 
   return {
     pages: pageMap,
-    site,
     redirects,
     unversionedTree,
     siteTree,

middleware/context.js

@@ -7,6 +7,7 @@ import productNames from '../lib/product-names.js'
 import warmServer from '../lib/warm-server.js'
 import searchVersions from '../lib/search/versions.js'
 import nonEnterpriseDefaultVersion from '../lib/non-enterprise-default-version.js'
+import { getDataByLanguage, getUIDataMerged } from '../lib/get-data.js'
 const activeProducts = Object.values(productMap).filter(
   (product) => !product.wip && !product.hidden
 )
@@ -26,7 +27,7 @@ const enterpriseServerVersions = Object.keys(allVersions).filter((version) =>
 // Note that additional middleware in middleware/index.js adds to this context object
 export default async function contextualize(req, res, next) {
   // Ensure that we load some data only once on first request
-  const { site, redirects, siteTree, pages: pageMap } = await warmServer()
+  const { redirects, siteTree, pages: pageMap } = await warmServer()
 
   req.context = {}
   req.context.process = { env: {} }
@@ -49,7 +50,12 @@ export default async function contextualize(req, res, next) {
   req.context.enterpriseServerReleases = enterpriseServerReleases
   req.context.enterpriseServerVersions = enterpriseServerVersions
   req.context.redirects = redirects
-  req.context.site = site[req.language].site
+  req.context.site = {
+    data: {
+      ui: getUIDataMerged(req.language),
+    },
+  }
+  req.context.getDottedData = (dottedPath) => getDataByLanguage(dottedPath, req.language)
   req.context.siteTree = siteTree
   req.context.pages = pageMap
   req.context.searchVersions = searchVersions

middleware/contextualizers/features.js

@@ -1,25 +1,43 @@
-import warmServer from '../../lib/warm-server.js'
 import getApplicableVersions from '../../lib/get-applicable-versions.js'
+import { getDeepDataByLanguage } from '../../lib/get-data.js'
 
-export default async function features(req, res, next) {
+export default function features(req, res, next) {
   if (!req.context.page) return next()
 
-  const { site } = await warmServer()
+  Object.entries(getFeaturesByVersion(req.context.currentVersion)).forEach(
-
+    ([featureName, isFeatureAvailableInCurrentVersion]) => {
-  // Determine whether the currentVersion belongs to the list of versions the feature is available in.
+      req.context[featureName] = isFeatureAvailableInCurrentVersion
-  // Note that we always exclusively use the English `data.features` because
+    }
-  // we don't want any of these translated (and possibly corrupt).
+  )
-  Object.keys(site.en.site.data.features).forEach((featureName) => {
-    const { versions } = site.en.site.data.features[featureName]
-    const applicableVersions = getApplicableVersions(versions, `data/features/${featureName}.yml`)
-
-    // Adding the resulting boolean to the context object gives us the ability to use
-    // `{% if featureName ... %}` conditionals in content files.
-    const isFeatureAvailableInCurrentVersion = applicableVersions.includes(
-      req.context.currentVersion
-    )
-    req.context[featureName] = isFeatureAvailableInCurrentVersion
-  })
 
   return next()
 }
+
+let allFeatures
+
+const cache = new Map()
+function getFeaturesByVersion(currentVersion) {
+  if (!cache.has(currentVersion)) {
+    if (!allFeatures) {
+      // As of Oct 2022, the `data/features/**` reading is *not* JIT.
+      // The `data/features` is deliberately not ignored in nodemon.json.
+      // See internal issue #2389
+      allFeatures = getDeepDataByLanguage('features', 'en')
+    }
+
+    const features = {}
+    // Determine whether the currentVersion belongs to the list of versions the feature is available in.
+    for (const [featureName, feature] of Object.entries(allFeatures)) {
+      const { versions } = feature
+      const applicableVersions = getApplicableVersions(versions, `data/features/${featureName}.yml`)
+
+      // Adding the resulting boolean to the context object gives us the ability to use
+      // `{% if featureName ... %}` conditionals in content files.
+      const isFeatureAvailableInCurrentVersion = applicableVersions.includes(currentVersion)
+      features[featureName] = isFeatureAvailableInCurrentVersion
+    }
+    cache.set(currentVersion, features)
+  }
+
+  return cache.get(currentVersion)
+}

middleware/contextualizers/ghae-release-notes.js

@@ -1,4 +1,5 @@
 import { formatReleases, renderPatchNotes } from '../../lib/release-notes-utils.js'
+import { getDeepDataByLanguage } from '../../lib/get-data.js'
 import { allVersions } from '../../lib/all-versions.js'
 
 export default async function ghaeReleaseNotesContext(req, res, next) {
@@ -9,7 +10,8 @@ export default async function ghaeReleaseNotesContext(req, res, next) {
   )
     return next()
 
-  const ghaeReleaseNotes = req.context.site.data['release-notes']['github-ae']
+  const ghaeReleaseNotes = getDeepDataByLanguage('release-notes.github-ae', req.language)
+
   // internalLatestRelease is set in lib/all-versions, e.g., '3.5' but UI still displays '@latest'.
   let requestedRelease = req.context.currentVersionObj.internalLatestRelease
 

middleware/contextualizers/ghes-release-notes.js

@@ -1,12 +1,13 @@
 import { formatReleases, renderPatchNotes } from '../../lib/release-notes-utils.js'
 import { all } from '../../lib/enterprise-server-releases.js'
+import { getDeepDataByLanguage } from '../../lib/get-data.js'
 
 export default async function ghesReleaseNotesContext(req, res, next) {
   if (!(req.pagePath.endsWith('/release-notes') || req.pagePath.endsWith('/admin'))) return next()
   const [requestedPlan, requestedRelease] = req.context.currentVersion.split('@')
   if (requestedPlan !== 'enterprise-server') return next()
 
-  const ghesReleaseNotes = req.context.site.data['release-notes']['enterprise-server']
+  const ghesReleaseNotes = getDeepDataByLanguage('release-notes.enterprise-server', req.language)
 
   // If the requested GHES release isn't found in data/release-notes/enterprise-server/*,
   // and it IS a valid GHES release, try being helpful and redirecting to the old location.

middleware/contextualizers/glossaries.js

@@ -1,7 +1,9 @@
+import { getDataByLanguage } from '../../lib/get-data.js'
+
 export default function glossaries(req, res, next) {
   if (!req.pagePath.endsWith('get-started/quickstart/github-glossary')) return next()
 
-  const glossaries = req.context.site.data.glossaries.external
+  const glossaries = getDataByLanguage('glossaries.external', req.context.currentLanguage)
   req.context.glossaries = glossaries.sort((a, b) => a.term.localeCompare(b.term))
 
   return next()

middleware/contextualizers/product-examples.js

@@ -1,20 +1,50 @@
 import getApplicableVersions from '../../lib/get-applicable-versions.js'
+import { getDataByLanguage } from '../../lib/get-data.js'
+
+function getProductExampleData(product, key, language) {
+  // Because getDataByLanguage() depends on reading data files from
+  // disk, asking for something that doesn't exist would throw a
+  // `ENOENT` error from `fs.readFile` but we want that to default
+  // to `undefined` because certain product's don't have all product
+  // examples.
+  try {
+    return getDataByLanguage(`product-examples.${product}.${key}`, language)
+  } catch (error) {
+    if (error.code === 'ENOENT') return
+    throw error
+  }
+}
 
 export default async function productExamples(req, res, next) {
   if (!req.context.page) return next()
   if (req.context.currentLayoutName !== 'product-landing') return next()
 
-  const productExamples = req.context.site.data['product-examples'][req.context.currentProduct]
+  const { currentProduct, currentLanguage } = req.context
-  if (!productExamples) return next()
+  if (currentProduct.includes('.'))
+    throw new Error(`currentProduct can not contain a . (${currentProduct})`)
 
-  req.context.productCommunityExamples = productExamples['community-examples']
+  req.context.productCommunityExamples = getProductExampleData(
-  req.context.productUserExamples = productExamples['user-examples']
+    currentProduct,
+    'community-examples',
+    currentLanguage
+  )
+  req.context.productUserExamples = getProductExampleData(
+    currentProduct,
+    'user-examples',
+    currentLanguage
+  )
+
+  const productCodeExamples = getProductExampleData(
+    currentProduct,
+    'code-examples',
+    currentLanguage
+  )
 
   // We currently only support versioning in code examples.
   // TODO support versioning across all example types.
   req.context.productCodeExamples =
-    productExamples['code-examples'] &&
+    productCodeExamples &&
-    productExamples['code-examples'].filter((example) => {
+    productCodeExamples.filter((example) => {
       // If an example block does NOT contain the versions prop, assume it's available in all versions
       return (
         !example.versions ||

middleware/index.js

@@ -269,7 +269,7 @@ export default function (app) {
   app.use(asyncMiddleware(instrument(currentProductTree, './contextualizers/current-product-tree')))
   app.use(asyncMiddleware(instrument(genericToc, './contextualizers/generic-toc')))
   app.use(instrument(breadcrumbs, './contextualizers/breadcrumbs'))
-  app.use(asyncMiddleware(instrument(features, './contextualizers/features')))
+  app.use(instrument(features, './contextualizers/features'))
   app.use(asyncMiddleware(instrument(productExamples, './contextualizers/product-examples')))
   app.use(asyncMiddleware(instrument(productGroups, './contextualizers/product-groups')))
   app.use(instrument(glossaries, './contextualizers/glossaries'))

middleware/learning-track.js

@@ -1,6 +1,7 @@
 import { getPathWithoutLanguage, getPathWithoutVersion } from '../lib/path-utils.js'
 import getLinkData from '../lib/get-link-data.js'
 import renderContent from '../lib/render-content/renderContent.js'
+import { getDeepDataByLanguage } from '../lib/get-data.js'
 
 export default async function learningTrack(req, res, next) {
   const noTrack = () => {
@@ -14,7 +15,8 @@ export default async function learningTrack(req, res, next) {
   if (!trackName) return noTrack()
 
   let trackProduct = req.context.currentProduct
-  let tracksPerProduct = req.context.site.data['learning-tracks'][trackProduct]
+  const allLearningTracks = getDeepDataByLanguage('learning-tracks', req.language)
+  let tracksPerProduct = allLearningTracks[trackProduct]
 
   // If there are no learning tracks for the current product, try and fall
   // back to the learning track product set as a URL parameter.  This handles
@@ -22,12 +24,11 @@ export default async function learningTrack(req, res, next) {
   // than the current learning track product.
   if (!tracksPerProduct) {
     trackProduct = req.query.learnProduct
-    tracksPerProduct = req.context.site.data['learning-tracks'][trackProduct]
+    tracksPerProduct = allLearningTracks[trackProduct]
   }
-
   if (!tracksPerProduct) return noTrack()
 
-  const track = req.context.site.data['learning-tracks'][trackProduct][trackName]
+  const track = allLearningTracks[trackProduct][trackName]
   if (!track) return noTrack()
 
   const currentLearningTrack = { trackName, trackProduct }

nodemon.json

@@ -6,6 +6,11 @@
     "translations",
     "stylesheets",
     "tests",
-    "content"
+    "content",
+    "data/reusables",
+    "data/variables",
+    "data/glossaries",
+    "data/product-examples",
+    "data/learning-tracks"
   ]
 }

tests/content/category-pages.js

@@ -6,7 +6,6 @@ import matter from '../../lib/read-frontmatter.js'
 import { zip, difference } from 'lodash-es'
 import GithubSlugger from 'github-slugger'
 import { decode } from 'html-entities'
-import loadSiteData from '../../lib/site-data.js'
 import renderContent from '../../lib/render-content/index.js'
 import getApplicableVersions from '../../lib/get-applicable-versions.js'
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
@@ -16,8 +15,6 @@ const slugger = new GithubSlugger()
 const contentDir = path.join(__dirname, '../../content')
 
 describe('category pages', () => {
-  const siteData = loadSiteData().en.site
-
   const walkOptions = {
     globs: ['*/index.md', 'enterprise/*/index.md'],
     ignore: [
@@ -100,7 +97,9 @@ describe('category pages', () => {
           })
 
           // Save the index title for later testing
-          indexTitle = await renderContent(data.title, { site: siteData }, { textOnly: true })
+          indexTitle = data.title.includes('{')
+            ? await renderContent(data.title, { currentLanguage: 'en' }, { textOnly: true })
+            : data.title
 
           publishedArticlePaths = (
             await Promise.all(

tests/content/glossary.js

@@ -1,7 +1,10 @@
-import loadSiteData from '../../lib/site-data.js'
+import { getDataByLanguage } from '../../lib/get-data'
 
 describe('glossaries', () => {
-  const glossaries = loadSiteData().en.site.data.glossaries
+  const glossaries = {
+    external: getDataByLanguage('glossaries.external', 'en'),
+    candidates: getDataByLanguage('glossaries.candidates', 'en'),
+  }
 
   test('are broken into external, internal, and candidates', async () => {
     const keys = Object.keys(glossaries)

tests/fixtures/liquid-tags/good-data-variable.md

@@ -3,4 +3,4 @@ title: Good sample page
 versions: '*'
 ---
 
-{% data variables.foo %}
+{% data variables.stuff.foo %}

tests/unit/liquid-helpers.js

@@ -1,33 +1,30 @@
-import { jest } from '@jest/globals'
+import { afterAll, jest, beforeAll, expect } from '@jest/globals'
 import { liquid } from '../../lib/render-content/index.js'
-import { loadPageMap } from '../../lib/page-data.js'
+import languages from '../../lib/languages.js'
-import nonEnterpriseDefaultVersion from '../../lib/non-enterprise-default-version.js'
+import { DataDirectory } from '../helpers/data-directory.js'
 
 describe('liquid helper tags', () => {
   jest.setTimeout(60 * 1000)
 
   const context = {}
-  let pageMap
+  let dd
-  beforeAll(async () => {
+  const enDirBefore = languages.en.dir
-    pageMap = await loadPageMap()
+
+  beforeAll(() => {
     context.currentLanguage = 'en'
-    context.currentVersion = nonEnterpriseDefaultVersion
+    dd = new DataDirectory({
-    context.pages = pageMap
-    context.redirects = {
-      '/en/desktop/contributing-and-collaborating-using-github-desktop': `/en/${nonEnterpriseDefaultVersion}/desktop/contributing-and-collaborating-using-github-desktop`,
-      '/en/desktop/contributing-and-collaborating-using-github-desktop/adding-and-cloning-repositories': `/en/${nonEnterpriseDefaultVersion}/desktop/contributing-and-collaborating-using-github-desktop/adding-and-cloning-repositories`,
-      '/en/github/writing-on-github/basic-writing-and-formatting-syntax': `/en/${nonEnterpriseDefaultVersion}/github/writing-on-github/basic-writing-and-formatting-syntax`,
-    }
-    context.site = {
       data: {
         reusables: {
           example: 'a rose by any other name\nwould smell as sweet',
         },
       },
-    }
+    })
-    context.page = {
+    languages.en.dir = dd.root
-      relativePath: 'desktop/index.md',
+  })
-    }
+
+  afterAll(() => {
+    dd.destroy()
+    languages.en.dir = enDirBefore
   })
 
   describe('indented_data_reference tag', () => {
@@ -63,44 +60,4 @@ would smell as sweet`
       expect(output).toBe(expected)
     })
   })
-
-  describe('data tag', () => {
-    test('handles bracketed array access within for-in loop', async () => {
-      const template = `
-{% for term in site.data.glossaries.external %}
-### {% data glossaries.external[forloop.index0].term %}
-{% data glossaries.external[forloop.index0].description %}
----
-{% endfor %}`
-
-      const localContext = { ...context }
-      localContext.site = {
-        data: {
-          variables: {
-            fire_emoji: ':fire:',
-          },
-          glossaries: {
-            external: [
-              { term: 'lit', description: 'Awesome things. {% data variables.fire_emoji %}' },
-              { term: 'Zhu Li', description: '_"Zhu Li, do the thing!"_ :point_up:' },
-            ],
-          },
-        },
-      }
-
-      const expected = `
-
-### lit
-Awesome things. :fire:
----
-
-### Zhu Li
-_"Zhu Li, do the thing!"_ :point_up:
----
-`
-
-      const output = await liquid.parseAndRender(template, localContext)
-      expect(output).toBe(expected)
-    })
-  })
 })

tests/unit/liquid-tags/data.js

@@ -1,12 +1,37 @@
-import { expect } from '@jest/globals'
-import path from 'path'
-import Page from '../../../lib/page.js'
-import nonEnterpriseDefaultVersion from '../../../lib/non-enterprise-default-version.js'
 import { fileURLToPath } from 'url'
+import path from 'path'
+
+import { afterAll, beforeAll, expect, describe, it } from '@jest/globals'
+
+import Page from '../../../lib/page.js'
+import languages from '../../../lib/languages.js'
+import nonEnterpriseDefaultVersion from '../../../lib/non-enterprise-default-version.js'
+import { DataDirectory } from '../../helpers/data-directory.js'
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 
 describe('data tag', () => {
+  let dd
+  const enDirBefore = languages.en.dir
+
+  beforeAll(() => {
+    dd = new DataDirectory({
+      data: {
+        variables: {
+          stuff: {
+            foo: 'Foo',
+          },
+        },
+      },
+    })
+    languages.en.dir = dd.root
+  })
+
+  afterAll(() => {
+    if (dd) dd.destroy()
+    languages.en.dir = enDirBefore
+  })
+
   it('should render fine if data is found', async () => {
     const page = await Page.init({
       relativePath: 'liquid-tags/good-data-variable.md',
@@ -18,22 +43,9 @@ describe('data tag', () => {
       currentLanguage: 'en',
       currentPath: '/en/liquid-tags/good-data-variable',
     }
-    const rendered = await page.render(
+    const rendered = await page.render(context)
-      Object.assign(
-        {
-          site: {
-            data: {
-              variables: {
-                foo: 'Foo',
-              },
-            },
-          },
-        },
-        context
-      )
-    )
     // The test fixture contains:
-    //   {% data variables.foo %}
+    //   {% data variables.stuff.foo %}
     // which we control the value of here in the test.
     expect(rendered.includes('Foo')).toBeTruthy()
   })
@@ -45,9 +57,10 @@ describe('data tag', () => {
     })
     const context = {
       currentPath: '/en/liquid-tags/bad-data-variable',
+      currentLanguage: 'en',
     }
     await expect(page.render(context)).rejects.toThrow(
-      "Can't find the key 'site.data.foo.bar.tipu' in the scope., line:2, col:1"
+      "Can't find the key 'foo.bar.tipu' in the scope., line:2, col:1"
     )
   })
 })