зеркало из https://github.com/github/docs.git
cache full rendering (#25424)
* cache full rendering * still not working with gzip * progress progress progress * smaller * hacky progress * small fixes * wip * lock file * wip * wip * package-lock updates * wip * search DOM in lowercase * simplify * with instrument * improve test coverage * mutateCheeriobodyByRequest * fix * remove renderContentCacheByContex * disable render caching in sync-search * diables things in github/github link checker * gzip lru * tidying up * updated * correct tests * fix: move userLanguage to LanguagesContext * Revert "fix: move userLanguage to LanguagesContext" This reverts commit d7c05d958c71eaad496eb46764eb845d80b866ca. * contexts ftw * fixed rendering tests * oops for got new file * nits addressed Co-authored-by: Mike Surowiec <mikesurowiec@users.noreply.github.com>
This commit is contained in:
Peter Bengtsson
GitHub
Родитель
00d0f82c8f
Коммит
18504871b9
|
|
@ -57,8 +57,11 @@ jobs:
|
|||
env:
|
||||
NODE_ENV: production
|
||||
PORT: 4000
|
||||
# Overload protection is on by default (when NODE_ENV==production)
|
||||
# but it would help in this context.
|
||||
DISABLE_OVERLOAD_PROTECTION: true
|
||||
DISABLE_RENDER_CACHING: true
|
||||
# Render caching won't help when we visit every page exactly once.
|
||||
DISABLE_RENDERING_CACHE: true
|
||||
run: |
|
||||
|
||||
node server.mjs &
|
||||
|
|
|
|||
|
|
@ -92,6 +92,8 @@ jobs:
|
|||
# Because the overload protection runs in NODE_ENV==production
|
||||
# and it can break the sync-search.
|
||||
DISABLE_OVERLOAD_PROTECTION: true
|
||||
# Render caching won't help when we visit every page exactly once.
|
||||
DISABLE_RENDERING_CACHE: true
|
||||
|
||||
run: npm run sync-search
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ export const DefaultLayout = (props: Props) => {
|
|||
const { t } = useTranslation(['errors', 'meta', 'scroll_button'])
|
||||
const router = useRouter()
|
||||
const metaDescription = page.introPlainText ? page.introPlainText : t('default_description')
|
||||
|
||||
return (
|
||||
<div className="d-lg-flex">
|
||||
<Head>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
import { createContext, useContext } from 'react'
|
||||
|
||||
export type DotComAuthenticatedContextT = {
|
||||
isDotComAuthenticated: boolean
|
||||
}
|
||||
|
||||
export const DotComAuthenticatedContext = createContext<DotComAuthenticatedContextT | null>(null)
|
||||
|
||||
export const useAuth = (): DotComAuthenticatedContextT => {
|
||||
const context = useContext(DotComAuthenticatedContext)
|
||||
|
||||
if (!context) {
|
||||
throw new Error(
|
||||
'"useAuthContext" may only be used inside "DotComAuthenticatedContext.Provider"'
|
||||
)
|
||||
}
|
||||
|
||||
return context
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ type LanguageItem = {
|
|||
|
||||
export type LanguagesContextT = {
|
||||
languages: Record<string, LanguageItem>
|
||||
userLanguage: string
|
||||
}
|
||||
|
||||
export const LanguagesContext = createContext<LanguagesContextT | null>(null)
|
||||
|
|
|
|||
|
|
@ -93,7 +93,6 @@ export type MainContextT = {
|
|||
relativePath?: string
|
||||
enterpriseServerReleases: EnterpriseServerReleases
|
||||
currentPathWithoutLanguage: string
|
||||
userLanguage: string
|
||||
allVersions: Record<string, VersionItem>
|
||||
currentVersion?: string
|
||||
currentProductTree?: ProductTreeNode | null
|
||||
|
|
@ -125,7 +124,6 @@ export type MainContextT = {
|
|||
|
||||
status: number
|
||||
fullUrl: string
|
||||
isDotComAuthenticated: boolean
|
||||
}
|
||||
|
||||
export const getMainContext = (req: any, res: any): MainContextT => {
|
||||
|
|
@ -181,7 +179,6 @@ export const getMainContext = (req: any, res: any): MainContextT => {
|
|||
'supported',
|
||||
]),
|
||||
enterpriseServerVersions: req.context.enterpriseServerVersions,
|
||||
userLanguage: req.context.userLanguage || '',
|
||||
allVersions: req.context.allVersions,
|
||||
currentVersion: req.context.currentVersion,
|
||||
currentProductTree: req.context.currentProductTree
|
||||
|
|
@ -192,7 +189,6 @@ export const getMainContext = (req: any, res: any): MainContextT => {
|
|||
nonEnterpriseDefaultVersion: req.context.nonEnterpriseDefaultVersion,
|
||||
status: res.statusCode,
|
||||
fullUrl: req.protocol + '://' + req.get('host') + req.originalUrl,
|
||||
isDotComAuthenticated: Boolean(req.cookies.dotcom_user),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { useVersion } from 'components/hooks/useVersion'
|
|||
|
||||
import { Link } from 'components/Link'
|
||||
import { useMainContext } from 'components/context/MainContext'
|
||||
import { useAuth } from 'components/context/DotComAuthenticatedContext'
|
||||
import { LanguagePicker } from './LanguagePicker'
|
||||
import { HeaderNotifications } from 'components/page-header/HeaderNotifications'
|
||||
import { ProductPicker } from 'components/page-header/ProductPicker'
|
||||
|
|
@ -17,7 +18,7 @@ import styles from './Header.module.scss'
|
|||
|
||||
export const Header = () => {
|
||||
const router = useRouter()
|
||||
const { isDotComAuthenticated, error } = useMainContext()
|
||||
const { error } = useMainContext()
|
||||
const { currentVersion } = useVersion()
|
||||
const { t } = useTranslation(['header', 'homepage'])
|
||||
const [isMenuOpen, setIsMenuOpen] = useState(
|
||||
|
|
@ -25,6 +26,8 @@ export const Header = () => {
|
|||
)
|
||||
const [scroll, setScroll] = useState(false)
|
||||
|
||||
const { isDotComAuthenticated } = useAuth()
|
||||
|
||||
const signupCTAVisible =
|
||||
!isDotComAuthenticated &&
|
||||
(currentVersion === 'free-pro-team@latest' || currentVersion === 'enterprise-cloud@latest')
|
||||
|
|
|
|||
|
|
@ -21,9 +21,9 @@ type Notif = {
|
|||
export const HeaderNotifications = () => {
|
||||
const router = useRouter()
|
||||
const { currentVersion } = useVersion()
|
||||
const { relativePath, allVersions, data, userLanguage, currentPathWithoutLanguage, page } =
|
||||
useMainContext()
|
||||
const { languages } = useLanguages()
|
||||
const { relativePath, allVersions, data, currentPathWithoutLanguage, page } = useMainContext()
|
||||
const { languages, userLanguage } = useLanguages()
|
||||
|
||||
const { t } = useTranslation('header')
|
||||
|
||||
const translationNotices: Array<Notif> = []
|
||||
|
|
|
|||
|
|
@ -1,11 +1,9 @@
|
|||
// export const defaultCSSThemeProps = {
|
||||
export const defaultCSSTheme = {
|
||||
colorMode: 'auto', // light, dark, auto
|
||||
nightTheme: 'dark',
|
||||
dayTheme: 'light',
|
||||
}
|
||||
|
||||
// export const defaultComponentThemeProps = {
|
||||
export const defaultComponentTheme = {
|
||||
colorMode: 'auto', // day, night, auto
|
||||
nightTheme: 'dark',
|
||||
|
|
|
|||
35
lib/page.js
35
lib/page.js
|
|
@ -24,22 +24,6 @@ import { union } from 'lodash-es'
|
|||
// every single time, we turn it into a Set once.
|
||||
const productMapKeysAsSet = new Set(Object.keys(productMap))
|
||||
|
||||
// Wrapper on renderContent() that caches the output depending on the
|
||||
// `context` by extracting information about the page's current permalink
|
||||
const _renderContentCache = new Map()
|
||||
|
||||
function renderContentCacheByContext(prefix) {
|
||||
return async function (template = '', context = {}, options = {}) {
|
||||
const { currentPath } = context
|
||||
const cacheKey = prefix + currentPath
|
||||
|
||||
if (!_renderContentCache.has(cacheKey)) {
|
||||
_renderContentCache.set(cacheKey, await renderContent(template, context, options))
|
||||
}
|
||||
return _renderContentCache.get(cacheKey)
|
||||
}
|
||||
}
|
||||
|
||||
class Page {
|
||||
static async init(opts) {
|
||||
opts = await Page.read(opts)
|
||||
|
|
@ -186,18 +170,18 @@ class Page {
|
|||
context.englishHeadings = englishHeadings
|
||||
}
|
||||
|
||||
this.intro = await renderContentCacheByContext('intro')(this.rawIntro, context)
|
||||
this.introPlainText = await renderContentCacheByContext('rawIntro')(this.rawIntro, context, {
|
||||
this.intro = await renderContent(this.rawIntro, context)
|
||||
this.introPlainText = await renderContent(this.rawIntro, context, {
|
||||
textOnly: true,
|
||||
})
|
||||
this.title = await renderContentCacheByContext('rawTitle')(this.rawTitle, context, {
|
||||
this.title = await renderContent(this.rawTitle, context, {
|
||||
textOnly: true,
|
||||
encodeEntities: true,
|
||||
})
|
||||
this.titlePlainText = await renderContentCacheByContext('titleText')(this.rawTitle, context, {
|
||||
this.titlePlainText = await renderContent(this.rawTitle, context, {
|
||||
textOnly: true,
|
||||
})
|
||||
this.shortTitle = await renderContentCacheByContext('shortTitle')(this.shortTitle, context, {
|
||||
this.shortTitle = await renderContent(this.shortTitle, context, {
|
||||
textOnly: true,
|
||||
encodeEntities: true,
|
||||
})
|
||||
|
|
@ -205,7 +189,7 @@ class Page {
|
|||
this.product_video = await renderContent(this.raw_product_video, context, { textOnly: true })
|
||||
|
||||
context.relativePath = this.relativePath
|
||||
const html = await renderContentCacheByContext('markdown')(this.markdown, context)
|
||||
const html = await renderContent(this.markdown, context)
|
||||
|
||||
// Adding communityRedirect for Discussions, Sponsors, and Codespaces - request from Product
|
||||
if (
|
||||
|
|
@ -222,15 +206,12 @@ class Page {
|
|||
|
||||
// product frontmatter may contain liquid
|
||||
if (this.rawProduct) {
|
||||
this.product = await renderContentCacheByContext('product')(this.rawProduct, context)
|
||||
this.product = await renderContent(this.rawProduct, context)
|
||||
}
|
||||
|
||||
// permissions frontmatter may contain liquid
|
||||
if (this.rawPermissions) {
|
||||
this.permissions = await renderContentCacheByContext('permissions')(
|
||||
this.rawPermissions,
|
||||
context
|
||||
)
|
||||
this.permissions = await renderContent(this.rawPermissions, context)
|
||||
}
|
||||
|
||||
// Learning tracks may contain Liquid and need to have versioning processed.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,167 @@
|
|||
import zlib from 'zlib'
|
||||
|
||||
import cheerio from 'cheerio'
|
||||
import QuickLRU from 'quick-lru'
|
||||
|
||||
// This is what NextJS uses when it injects the JSON serialized
|
||||
// in the `<script id="__NEXT_DATA__">`
|
||||
import { htmlEscapeJsonString } from 'next/dist/server/htmlescape.js'
|
||||
|
||||
import { getTheme } from '../lib/get-theme.js'
|
||||
import statsd from '../lib/statsd.js'
|
||||
|
||||
const HEADER_NAME = 'x-middleware-cache'
|
||||
const HEADER_VALUE_HIT = 'hit'
|
||||
const HEADER_VALUE_MISS = 'miss'
|
||||
const HEADER_VALUE_DISABLED = 'disabled'
|
||||
const HEADER_VALUE_TRANSFERRING = 'transferring'
|
||||
|
||||
const DISABLE_RENDERING_CACHE = Boolean(JSON.parse(process.env.DISABLE_RENDERING_CACHE || 'false'))
|
||||
|
||||
const cheerioCache = new QuickLRU({
|
||||
// NOTE: Apr 20, when storing about 200 cheerio instances, the total
|
||||
// heap size becomes about 2.3GB.
|
||||
maxSize: 100,
|
||||
// Don't use arrow function so we can access `this`.
|
||||
onEviction: function onEviction() {
|
||||
const { heapUsed } = process.memoryUsage()
|
||||
statsd.gauge('rendering_cache_cheerio', heapUsed, [`size:${this.size}`])
|
||||
},
|
||||
})
|
||||
|
||||
const gzipCache = new QuickLRU({
|
||||
maxSize: 1000,
|
||||
// Don't use arrow function so we can access `this`.
|
||||
onEviction: function onEviction() {
|
||||
const { heapUsed } = process.memoryUsage()
|
||||
statsd.gauge('rendering_cache_gzip', heapUsed, [`size:${gzipCache.size}`])
|
||||
},
|
||||
})
|
||||
|
||||
export default async function cacheFullRendering(req, res, next) {
|
||||
// Even if you use `app.get('/*', myMiddleware)` in Express, the
|
||||
// middleware will be executed for HEAD requests.
|
||||
if (req.method !== 'GET') return next()
|
||||
|
||||
// The req.pagePath will be identical if it's a regular HTML GET
|
||||
// or one of those /_next/data/... URLs.
|
||||
const key = req.url
|
||||
|
||||
// We have 2 LRU caches.
|
||||
// - Tuples of [cheerio object, headers]
|
||||
// - Tuples of [html gzipped, headers]
|
||||
// The reason for having two is that many cheerio objects will
|
||||
// significantly bloat the heap memory. Where as storing the
|
||||
// html strings as gzip buffers is tiny.
|
||||
// The point of using cheerio objects, is to avoid deserializing the
|
||||
// HTML on every warm hit (e.g. stampeding herd) and only pay
|
||||
// for the mutation + serialization which is unavoidable.
|
||||
// Since the gzip cache is larger than the cheerio cache,
|
||||
// we elevate from one cache to the other. Like layers of caching.
|
||||
|
||||
if (!cheerioCache.has(key) && gzipCache.has(key)) {
|
||||
res.setHeader(HEADER_NAME, HEADER_VALUE_TRANSFERRING)
|
||||
const [htmlBuffer, headers] = gzipCache.get(key)
|
||||
setHeaders(headers, res)
|
||||
const html = zlib.gunzipSync(htmlBuffer).toString()
|
||||
const body = cheerio.load(html)
|
||||
cheerioCache.set(key, [body, headers])
|
||||
mutateCheeriobodyByRequest(body, req)
|
||||
return res.status(200).send(body.html())
|
||||
} else if (cheerioCache.has(key)) {
|
||||
res.setHeader(HEADER_NAME, HEADER_VALUE_HIT)
|
||||
const [$, headers] = cheerioCache.get(key)
|
||||
setHeaders(headers, res)
|
||||
mutateCheeriobodyByRequest($, req)
|
||||
return res.status(200).send($.html())
|
||||
} else {
|
||||
res.setHeader(HEADER_NAME, HEADER_VALUE_MISS)
|
||||
}
|
||||
|
||||
if (DISABLE_RENDERING_CACHE) {
|
||||
res.setHeader(HEADER_NAME, HEADER_VALUE_DISABLED)
|
||||
} else {
|
||||
const originalEndFunc = res.end.bind(res)
|
||||
res.end = function (body) {
|
||||
if (body && res.statusCode === 200) {
|
||||
// It's important to note that we only cache the HTML outputs.
|
||||
// Why, because JSON outputs should be cached in the CDN.
|
||||
// The only JSON outputs we have today is the search API
|
||||
// and the NextJS data requests. These are not dependent on the
|
||||
// request cookie, so they're primed for caching in the CDN.
|
||||
const ct = res.get('content-type')
|
||||
if (ct.startsWith('text/html')) {
|
||||
const $ = cheerio.load(body)
|
||||
const headers = res.getHeaders()
|
||||
cheerioCache.set(key, [$, headers])
|
||||
const gzipped = zlib.gzipSync(Buffer.from(body))
|
||||
gzipCache.set(key, [gzipped, headers])
|
||||
}
|
||||
// If it's not HTML or JSON, it's probably an image (binary)
|
||||
// or some plain text. Let's ignore all of those.
|
||||
}
|
||||
return originalEndFunc(body)
|
||||
}
|
||||
}
|
||||
|
||||
next()
|
||||
}
|
||||
|
||||
function setHeaders(headers, res) {
|
||||
Object.entries(headers).forEach(([key, value]) => {
|
||||
if (!(key === HEADER_NAME || key === 'set-cookie')) {
|
||||
res.setHeader(key, value)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function mutateCheeriobodyByRequest($, req) {
|
||||
// A fresh CSRF token into the <meta> tag
|
||||
const freshCsrfToken = req.csrfToken()
|
||||
$('meta[name="csrf-token"]').attr('content', freshCsrfToken)
|
||||
|
||||
// Populate if you have the `dotcom_user` user cookie and it's truthy
|
||||
const isDotComAuthenticated = Boolean(req.cookies?.dotcom_user)
|
||||
|
||||
const cssTheme = getTheme(req, true)
|
||||
const theme = getTheme(req, false)
|
||||
|
||||
// The <body> needs tags pertaining to the parsed theme
|
||||
// Don't use `$.data()` because it doesn't actually mutate the "DOM"
|
||||
// https://github.com/cheeriojs/cheerio/issues/950#issuecomment-274324269
|
||||
$('body')
|
||||
.attr('data-color-mode', cssTheme.colorMode)
|
||||
.attr('data-dark-theme', cssTheme.nightTheme)
|
||||
.attr('data-light-theme', cssTheme.dayTheme)
|
||||
|
||||
// Update the __NEXT_DATA__ too with the equivalent pieces
|
||||
const nextData = $('script#__NEXT_DATA__')
|
||||
console.assert(nextData.length === 1, 'Not exactly 1')
|
||||
// Note, once we upgrade to cheerio >= v1.0.0-rc.11
|
||||
// we can access this with `.text()`.
|
||||
// See https://github.com/cheeriojs/cheerio/releases/tag/v1.0.0-rc.11
|
||||
// and https://github.com/cheeriojs/cheerio/pull/2509
|
||||
const parsedNextData = JSON.parse(nextData.get()[0].children[0].data)
|
||||
parsedNextData.props.csrfToken = freshCsrfToken
|
||||
parsedNextData.props.dotComAuthenticatedContext.isDotComAuthenticated = isDotComAuthenticated
|
||||
parsedNextData.props.languagesContext.userLanguage = req.context.userLanguage
|
||||
parsedNextData.props.themeProps = {
|
||||
colorMode: theme.colorMode,
|
||||
nightTheme: theme.nightTheme,
|
||||
dayTheme: theme.dayTheme,
|
||||
}
|
||||
nextData.text(htmlEscapeJsonString(JSON.stringify(parsedNextData)))
|
||||
|
||||
// The <ThemeProvider {...} preventSSRMismatch> component will
|
||||
// inject a script tag too that looks like this:
|
||||
//
|
||||
// <script
|
||||
// type="application/json"
|
||||
// id="__PRIMER_DATA__">{"resolvedServerColorMode":"night"}</script>
|
||||
//
|
||||
const primerData = $('script#__PRIMER_DATA__')
|
||||
console.assert(primerData.length === 1, 'Not exactly 1')
|
||||
const parsedPrimerData = JSON.parse(primerData.get()[0].children[0].data)
|
||||
parsedPrimerData.resolvedServerColorMode = cssTheme.colorMode === 'dark' ? 'night' : 'day'
|
||||
primerData.text(htmlEscapeJsonString(JSON.stringify(parsedPrimerData)))
|
||||
}
|
||||
|
|
@ -63,6 +63,7 @@ import assetPreprocessing from './asset-preprocessing.js'
|
|||
import archivedAssetRedirects from './archived-asset-redirects.js'
|
||||
import favicons from './favicons.js'
|
||||
import setStaticAssetCaching from './static-asset-caching.js'
|
||||
import cacheFullRendering from './cache-full-rendering.js'
|
||||
import protect from './overload-protection.js'
|
||||
import fastHead from './fast-head.js'
|
||||
|
||||
|
|
@ -317,6 +318,10 @@ export default function (app) {
|
|||
// full page rendering.
|
||||
app.head('/*', fastHead)
|
||||
|
||||
// For performance, this is before contextualizers if, on a cache hit,
|
||||
// we can't reuse a rendered response without having to contextualize.
|
||||
app.get('/*', asyncMiddleware(instrument(cacheFullRendering, './cache-full-rendering')))
|
||||
|
||||
// *** Preparation for render-page: contextualizers ***
|
||||
app.use(asyncMiddleware(instrument(releaseNotes, './contextualizers/release-notes')))
|
||||
app.use(instrument(graphQL, './contextualizers/graphql'))
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ export default async function renderPage(req, res, next) {
|
|||
|
||||
// Just finish fast without all the details like Content-Length
|
||||
if (req.method === 'HEAD') {
|
||||
return res.status(200).end()
|
||||
return res.status(200).send('')
|
||||
}
|
||||
|
||||
// Updating the Last-Modified header for substantive changes on a page for engineering
|
||||
|
|
|
|||
|
|
@ -39,4 +39,7 @@ module.exports = {
|
|||
config.experiments.topLevelAwait = true
|
||||
return config
|
||||
},
|
||||
|
||||
// https://nextjs.org/docs/api-reference/next.config.js/compression
|
||||
compress: false,
|
||||
}
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -62,6 +62,7 @@
|
|||
"overload-protection": "^1.2.3",
|
||||
"parse5": "^6.0.1",
|
||||
"port-used": "^2.0.8",
|
||||
"quick-lru": "6.1.0",
|
||||
"react": "^17.0.2",
|
||||
"react-dom": "^17.0.2",
|
||||
"react-markdown": "^8.0.0",
|
||||
|
|
|
|||
|
|
@ -10,14 +10,27 @@ import '../stylesheets/index.scss'
|
|||
import events from 'components/lib/events'
|
||||
import experiment from 'components/lib/experiment'
|
||||
import { LanguagesContext, LanguagesContextT } from 'components/context/LanguagesContext'
|
||||
import {
|
||||
DotComAuthenticatedContext,
|
||||
DotComAuthenticatedContextT,
|
||||
} from 'components/context/DotComAuthenticatedContext'
|
||||
import { defaultComponentTheme } from 'lib/get-theme.js'
|
||||
|
||||
type MyAppProps = AppProps & {
|
||||
csrfToken: string
|
||||
isDotComAuthenticated: boolean
|
||||
themeProps: typeof defaultComponentTheme & Pick<ThemeProviderProps, 'colorMode'>
|
||||
languagesContext: LanguagesContextT
|
||||
dotComAuthenticatedContext: DotComAuthenticatedContextT
|
||||
}
|
||||
const MyApp = ({ Component, pageProps, csrfToken, themeProps, languagesContext }: MyAppProps) => {
|
||||
const MyApp = ({
|
||||
Component,
|
||||
pageProps,
|
||||
csrfToken,
|
||||
themeProps,
|
||||
languagesContext,
|
||||
dotComAuthenticatedContext,
|
||||
}: MyAppProps) => {
|
||||
useEffect(() => {
|
||||
events()
|
||||
experiment()
|
||||
|
|
@ -58,7 +71,9 @@ const MyApp = ({ Component, pageProps, csrfToken, themeProps, languagesContext }
|
|||
preventSSRMismatch
|
||||
>
|
||||
<LanguagesContext.Provider value={languagesContext}>
|
||||
<Component {...pageProps} />
|
||||
<DotComAuthenticatedContext.Provider value={dotComAuthenticatedContext}>
|
||||
<Component {...pageProps} />
|
||||
</DotComAuthenticatedContext.Provider>
|
||||
</LanguagesContext.Provider>
|
||||
</ThemeProvider>
|
||||
</SSRProvider>
|
||||
|
|
@ -66,6 +81,10 @@ const MyApp = ({ Component, pageProps, csrfToken, themeProps, languagesContext }
|
|||
)
|
||||
}
|
||||
|
||||
// Remember, function is only called once if the rendered page can
|
||||
// be in-memory cached. But still, the `<MyApp>` component will be
|
||||
// executed every time **in the client** if it was the first time
|
||||
// ever (since restart) or from a cached HTML.
|
||||
MyApp.getInitialProps = async (appContext: AppContext) => {
|
||||
const { ctx } = appContext
|
||||
// calls page's `getInitialProps` and fills `appProps.pageProps`
|
||||
|
|
@ -78,7 +97,8 @@ MyApp.getInitialProps = async (appContext: AppContext) => {
|
|||
...appProps,
|
||||
themeProps: getTheme(req),
|
||||
csrfToken: req?.csrfToken?.() || '',
|
||||
languagesContext: { languages: req.context.languages },
|
||||
languagesContext: { languages: req.context.languages, userLanguage: req.context.userLanguage },
|
||||
dotComAuthenticatedContext: { isDotComAuthenticated: Boolean(req.cookies?.dotcom_user) },
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import LunrIndex from './lunr-search-index.js'
|
|||
// Build a search data file for every combination of product version and language
|
||||
// e.g. `github-docs-dotcom-en.json` and `github-docs-2.14-ja.json`
|
||||
export default async function syncSearchIndexes(opts = {}) {
|
||||
const t0 = new Date()
|
||||
if (opts.language) {
|
||||
if (!Object.keys(languages).includes(opts.language)) {
|
||||
console.log(
|
||||
|
|
@ -89,6 +90,9 @@ export default async function syncSearchIndexes(opts = {}) {
|
|||
}
|
||||
}
|
||||
}
|
||||
const t1 = new Date()
|
||||
const tookSec = (t1.getTime() - t0.getTime()) / 1000
|
||||
|
||||
console.log('\nDone!')
|
||||
console.log(`Took ${tookSec.toFixed(1)} seconds`)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
const NEXT_DATA_QUERY = 'script#__NEXT_DATA__'
|
||||
const PRIMER_DATA_QUERY = 'script#__PRIMER_DATA__'
|
||||
|
||||
function getScriptData($, key) {
|
||||
const data = $(key)
|
||||
if (!data.length === 1) {
|
||||
throw new Error(`Not exactly 1 element match for '${key}'. Found ${data.length}`)
|
||||
}
|
||||
return JSON.parse(data.get()[0].children[0].data)
|
||||
}
|
||||
|
||||
export const getNextData = ($) => getScriptData($, NEXT_DATA_QUERY)
|
||||
export const getPrimerData = ($) => getScriptData($, PRIMER_DATA_QUERY)
|
||||
|
||||
export const getUserLanguage = ($) => {
|
||||
// Because the page might come from the middleware rendering cache,
|
||||
// the DOM won't get updated until the first client-side React render.
|
||||
// But we can assert the data that would be used for that first render.
|
||||
const { props } = getNextData($)
|
||||
return props.languagesContext.userLanguage
|
||||
}
|
||||
|
||||
export const getIsDotComAuthenticated = ($) => {
|
||||
// Because the page might come from the middleware rendering cache,
|
||||
// the DOM won't get updated until the first client-side React render.
|
||||
// But we can assert the data that would be used for that first render.
|
||||
const { props } = getNextData($)
|
||||
return props.dotComAuthenticatedContext.isDotComAuthenticated
|
||||
}
|
||||
|
|
@ -13,7 +13,16 @@ describe('<head>', () => {
|
|||
expect($hreflangs.length).toEqual(Object.keys(languages).length)
|
||||
expect($('link[href="https://docs.github.com/cn"]').length).toBe(1)
|
||||
expect($('link[href="https://docs.github.com/ja"]').length).toBe(1)
|
||||
expect($('link[hrefLang="en"]').length).toBe(1)
|
||||
// Due to a bug in either NextJS, JSX, or TypeScript,
|
||||
// when put `<link hrefLang="xxx">` in a .tsx file, this incorrectly
|
||||
// gets rendered out as `<link hrefLang="xxx">` in the final HTML.
|
||||
// Note the uppercase L. It's supposed to become `<link hreflang="xxx">`.
|
||||
// When cheerio serializes to HTML, it gets this right so it lowercases
|
||||
// the attribute. So if this rendering in this jest test was the first
|
||||
// ever cold hit, you might get the buggy HTML from React or you
|
||||
// might get the correct HTML from cheerio's `.html()` serializer.
|
||||
// This is why we're looking for either.
|
||||
expect($('link[hreflang="en"]').length + $('link[hrefLang="en"]').length).toBe(1)
|
||||
})
|
||||
|
||||
test('includes page intro in `description` meta tag', async () => {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import { jest } from '@jest/globals'
|
||||
import { expect, jest } from '@jest/globals'
|
||||
|
||||
import { getDOM } from '../helpers/e2etest.js'
|
||||
import { oldestSupported } from '../../lib/enterprise-server-releases.js'
|
||||
import { getUserLanguage } from '../helpers/script-data.js'
|
||||
|
||||
describe('header', () => {
|
||||
jest.setTimeout(5 * 60 * 1000)
|
||||
|
|
@ -91,54 +92,31 @@ describe('header', () => {
|
|||
test("renders a link to the same page in user's preferred language, if available", async () => {
|
||||
const headers = { 'accept-language': 'ja' }
|
||||
const $ = await getDOM('/en', { headers })
|
||||
expect($('[data-testid=header-notification][data-type=TRANSLATION]').length).toBe(1)
|
||||
expect($('[data-testid=header-notification] a[href*="/ja"]').length).toBe(1)
|
||||
expect(getUserLanguage($)).toBe('ja')
|
||||
})
|
||||
|
||||
test("renders a link to the same page if user's preferred language is Chinese - PRC", async () => {
|
||||
const headers = { 'accept-language': 'zh-CN' }
|
||||
const $ = await getDOM('/en', { headers })
|
||||
expect($('[data-testid=header-notification][data-type=TRANSLATION]').length).toBe(1)
|
||||
expect($('[data-testid=header-notification] a[href*="/cn"]').length).toBe(1)
|
||||
})
|
||||
|
||||
test("does not render a link when user's preferred language is Chinese - Taiwan", async () => {
|
||||
const headers = { 'accept-language': 'zh-TW' }
|
||||
const $ = await getDOM('/en', { headers })
|
||||
expect($('[data-testid=header-notification]').length).toBe(0)
|
||||
})
|
||||
|
||||
test("does not render a link when user's preferred language is English", async () => {
|
||||
const headers = { 'accept-language': 'en' }
|
||||
const $ = await getDOM('/en', { headers })
|
||||
expect($('[data-testid=header-notification]').length).toBe(0)
|
||||
expect(getUserLanguage($)).toBe('cn')
|
||||
})
|
||||
|
||||
test("renders a link to the same page in user's preferred language from multiple, if available", async () => {
|
||||
const headers = { 'accept-language': 'ja, *;q=0.9' }
|
||||
const $ = await getDOM('/en', { headers })
|
||||
expect($('[data-testid=header-notification][data-type=TRANSLATION]').length).toBe(1)
|
||||
expect($('[data-testid=header-notification] a[href*="/ja"]').length).toBe(1)
|
||||
expect(getUserLanguage($)).toBe('ja')
|
||||
})
|
||||
|
||||
test("renders a link to the same page in user's preferred language with weights, if available", async () => {
|
||||
const headers = { 'accept-language': 'ja;q=1.0, *;q=0.9' }
|
||||
const $ = await getDOM('/en', { headers })
|
||||
expect($('[data-testid=header-notification][data-type=TRANSLATION]').length).toBe(1)
|
||||
expect($('[data-testid=header-notification] a[href*="/ja"]').length).toBe(1)
|
||||
expect(getUserLanguage($)).toBe('ja')
|
||||
})
|
||||
|
||||
test("renders a link to the user's 2nd preferred language if 1st is not available", async () => {
|
||||
const headers = { 'accept-language': 'zh-TW,zh;q=0.9,ja *;q=0.8' }
|
||||
const $ = await getDOM('/en', { headers })
|
||||
expect($('[data-testid=header-notification][data-type=TRANSLATION]').length).toBe(1)
|
||||
expect($('[data-testid=header-notification] a[href*="/ja"]').length).toBe(1)
|
||||
})
|
||||
|
||||
test('renders no notices if no language preference is available', async () => {
|
||||
const headers = { 'accept-language': 'zh-TW,zh;q=0.9,zh-SG *;q=0.8' }
|
||||
const $ = await getDOM('/en', { headers })
|
||||
expect($('[data-testid=header-notification]').length).toBe(0)
|
||||
expect(getUserLanguage($)).toBe('ja')
|
||||
})
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,160 @@
|
|||
import cheerio from 'cheerio'
|
||||
import { expect, jest, test } from '@jest/globals'
|
||||
|
||||
import { get } from '../helpers/e2etest.js'
|
||||
import { PREFERRED_LOCALE_COOKIE_NAME } from '../../middleware/detect-language.js'
|
||||
import {
|
||||
getNextData,
|
||||
getPrimerData,
|
||||
getUserLanguage,
|
||||
getIsDotComAuthenticated,
|
||||
} from '../helpers/script-data.js'
|
||||
|
||||
const serializeTheme = (theme) => {
|
||||
return encodeURIComponent(JSON.stringify(theme))
|
||||
}
|
||||
|
||||
describe('in-memory render caching', () => {
|
||||
jest.setTimeout(30 * 1000)
|
||||
|
||||
test('second render should be a cache hit with different csrf-token', async () => {
|
||||
const res = await get('/en')
|
||||
// Because these are effectively end-to-end tests, you can't expect
|
||||
// the first request to be a cache miss because another end-to-end
|
||||
// test might have "warmed up" this endpoint.
|
||||
expect(res.headers['x-middleware-cache']).toBeTruthy()
|
||||
const $1 = cheerio.load(res.text)
|
||||
const res2 = await get('/en')
|
||||
expect(res2.headers['x-middleware-cache']).toBe('hit')
|
||||
const $2 = cheerio.load(res2.text)
|
||||
const csrfTokenHTML1 = $1('meta[name="csrf-token"]').attr('content')
|
||||
const csrfTokenHTML2 = $2('meta[name="csrf-token"]').attr('content')
|
||||
expect(csrfTokenHTML1).not.toBe(csrfTokenHTML2)
|
||||
// The HTML is one thing, we also need to check that the
|
||||
// __NEXT_DATA__ serialized (JSON) state is different.
|
||||
const csrfTokenNEXT1 = getNextData($1).props.csrfToken
|
||||
const csrfTokenNEXT2 = getNextData($2).props.csrfToken
|
||||
expect(csrfTokenHTML1).toBe(csrfTokenNEXT1)
|
||||
expect(csrfTokenHTML2).toBe(csrfTokenNEXT2)
|
||||
expect(csrfTokenNEXT1).not.toBe(csrfTokenNEXT2)
|
||||
})
|
||||
|
||||
test('second render should be a cache hit with different dotcom-auth', async () => {
|
||||
// Anonymous first
|
||||
const res = await get('/en')
|
||||
// Because these are effectively end-to-end tests, you can't expect
|
||||
// the first request to be a cache miss because another end-to-end
|
||||
// test might have "warmed up" this endpoint.
|
||||
expect(res.headers['x-middleware-cache']).toBeTruthy()
|
||||
const $1 = cheerio.load(res.text)
|
||||
const res2 = await get('/en', {
|
||||
headers: {
|
||||
cookie: 'dotcom_user=peterbe',
|
||||
},
|
||||
})
|
||||
expect(res2.headers['x-middleware-cache']).toBe('hit')
|
||||
const $2 = cheerio.load(res2.text)
|
||||
// The HTML is one thing, we also need to check that the
|
||||
// __NEXT_DATA__ serialized (JSON) state is different.
|
||||
const dotcomAuthNEXT1 = getIsDotComAuthenticated($1)
|
||||
const dotcomAuthNEXT2 = getIsDotComAuthenticated($2)
|
||||
expect(dotcomAuthNEXT1).not.toBe(dotcomAuthNEXT2)
|
||||
})
|
||||
|
||||
test('second render should be a cache hit with different theme properties', async () => {
|
||||
const cookieValue1 = {
|
||||
color_mode: 'light',
|
||||
light_theme: { name: 'light', color_mode: 'light' },
|
||||
dark_theme: { name: 'dark_high_contrast', color_mode: 'dark' },
|
||||
}
|
||||
// Light mode first
|
||||
const res1 = await get('/en', {
|
||||
headers: {
|
||||
cookie: `color_mode=${serializeTheme(cookieValue1)}`,
|
||||
},
|
||||
})
|
||||
// Because these are effectively end-to-end tests, you can't expect
|
||||
// the first request to be a cache miss because another end-to-end
|
||||
// test might have "warmed up" this endpoint.
|
||||
expect(res1.headers['x-middleware-cache']).toBeTruthy()
|
||||
const $1 = cheerio.load(res1.text)
|
||||
expect($1('body').data('color-mode')).toBe(cookieValue1.color_mode)
|
||||
const themeProps1 = getNextData($1).props.themeProps
|
||||
expect(themeProps1.colorMode).toBe('day')
|
||||
|
||||
const cookieValue2 = {
|
||||
color_mode: 'dark',
|
||||
light_theme: { name: 'light', color_mode: 'light' },
|
||||
dark_theme: { name: 'dark_high_contrast', color_mode: 'dark' },
|
||||
}
|
||||
const res2 = await get('/en', {
|
||||
headers: {
|
||||
cookie: `color_mode=${serializeTheme(cookieValue2)}`,
|
||||
},
|
||||
})
|
||||
expect(res2.headers['x-middleware-cache']).toBeTruthy()
|
||||
const $2 = cheerio.load(res2.text)
|
||||
expect($2('body').data('color-mode')).toBe(cookieValue2.color_mode)
|
||||
const themeProps2 = getNextData($2).props.themeProps
|
||||
expect(themeProps2.colorMode).toBe('night')
|
||||
})
|
||||
|
||||
test('second render should be cache hit with different resolvedServerColorMode in __PRIMER_DATA__', async () => {
|
||||
await get('/en') // first render to assert the next render comes from cache
|
||||
|
||||
const res = await get('/en', {
|
||||
headers: {
|
||||
cookie: `color_mode=${serializeTheme({
|
||||
color_mode: 'dark',
|
||||
light_theme: { name: 'light', color_mode: 'light' },
|
||||
dark_theme: { name: 'dark_high_contrast', color_mode: 'dark' },
|
||||
})}`,
|
||||
},
|
||||
})
|
||||
expect(res.headers['x-middleware-cache']).toBeTruthy()
|
||||
const $ = cheerio.load(res.text)
|
||||
const data = getPrimerData($)
|
||||
expect(data.resolvedServerColorMode).toBe('night')
|
||||
|
||||
// Now do it all over again but with a light color mode
|
||||
const res2 = await get('/en', {
|
||||
headers: {
|
||||
cookie: `color_mode=${serializeTheme({
|
||||
color_mode: 'light',
|
||||
light_theme: { name: 'light', color_mode: 'light' },
|
||||
dark_theme: { name: 'dark_high_contrast', color_mode: 'dark' },
|
||||
})}`,
|
||||
},
|
||||
})
|
||||
expect(res2.headers['x-middleware-cache']).toBeTruthy()
|
||||
const $2 = cheerio.load(res2.text)
|
||||
const data2 = getPrimerData($2)
|
||||
expect(data2.resolvedServerColorMode).toBe('day')
|
||||
})
|
||||
|
||||
test('user-language, by header, in meta tag', async () => {
|
||||
await get('/en') // first render to assert the next render comes from cache
|
||||
|
||||
const res = await get('/en', {
|
||||
headers: { 'accept-language': 'ja;q=1.0, *;q=0.9' },
|
||||
})
|
||||
expect(res.headers['x-middleware-cache']).toBeTruthy()
|
||||
const $ = cheerio.load(res.text)
|
||||
const userLanguage = getUserLanguage($)
|
||||
expect(userLanguage).toBe('ja')
|
||||
})
|
||||
|
||||
test('user-language, by cookie, in meta tag', async () => {
|
||||
await get('/en') // first render to assert the next render comes from cache
|
||||
|
||||
const res = await get('/en', {
|
||||
headers: {
|
||||
Cookie: `${PREFERRED_LOCALE_COOKIE_NAME}=ja`,
|
||||
},
|
||||
})
|
||||
expect(res.headers['x-middleware-cache']).toBeTruthy()
|
||||
const $ = cheerio.load(res.text)
|
||||
const userLanguage = getUserLanguage($)
|
||||
expect(userLanguage).toBe('ja')
|
||||
})
|
||||
})
|
||||
|
|
@ -1,23 +1,14 @@
|
|||
import { jest, describe, expect } from '@jest/globals'
|
||||
|
||||
import { getDOM } from '../helpers/e2etest.js'
|
||||
import { getIsDotComAuthenticated } from '../helpers/script-data.js'
|
||||
|
||||
describe('GHEC sign up button', () => {
|
||||
jest.setTimeout(60 * 1000)
|
||||
|
||||
test('present by default', async () => {
|
||||
test('false by default', async () => {
|
||||
const $ = await getDOM('/en')
|
||||
expect($('a[href^="https://github.com/signup"]').length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
test('present on enterprise-cloud pages', async () => {
|
||||
const $ = await getDOM('/en/enterprise-cloud@latest')
|
||||
expect($('a[href^="https://github.com/signup"]').length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
test('not present on enterprise-server pages', async () => {
|
||||
const $ = await getDOM('/en/enterprise-server@latest')
|
||||
expect($('a[href^="https://github.com/signup"]').length).toBe(0)
|
||||
expect(getIsDotComAuthenticated($)).toBe(false)
|
||||
})
|
||||
|
||||
test('not present if dotcom_user cookie', async () => {
|
||||
|
|
@ -26,6 +17,15 @@ describe('GHEC sign up button', () => {
|
|||
cookie: 'dotcom_user=peterbe',
|
||||
},
|
||||
})
|
||||
expect($('a[href^="https://github.com/signup"]').length).toBe(0)
|
||||
expect(getIsDotComAuthenticated($)).toBe(true)
|
||||
|
||||
// Do another request, same URL, but different cookie, just to
|
||||
// make sure the server-side rendering cache isn't failing.
|
||||
const $2 = await getDOM('/en', {
|
||||
headers: {
|
||||
cookie: 'bla=bla',
|
||||
},
|
||||
})
|
||||
expect(getIsDotComAuthenticated($2)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче