feat: add nextjs middleware handling (#19139)

* feat: add nextjs middleware handling split

* fix: eslint errors

* fix: filter boolean from csp list

* fix: feature flag nextjs server start

* feat: add prettier rules for ts,tsx files

* fix: remove unnecessary async from next middleware

* fix: next middleware name

* Update tsconfig.json

Co-authored-by: James M. Greene <JamesMGreene@github.com>

* Update next-env.d.ts

Co-authored-by: James M. Greene <JamesMGreene@github.com>

* fix: add typescript linting to lint command

* add comment for unsafe-eval, update webpack to use eval in development

* fix: feature flag typo

Co-authored-by: James M. Greene <JamesMGreene@github.com>

.gitignore

@@ -10,6 +10,7 @@ coverage/
 /content/early-access
 /data/early-access
 dist
+.next
 
 # blc: broken link checker
 blc_output.log

.prettierrc.json

@@ -1,12 +1,20 @@
 {
   "overrides": [
     {
-      "files":[
-        "**/*.{yml,yaml}"
-      ],
+      "files": ["**/*.{yml,yaml}"],
       "options": {
         "singleQuote": true
       }
+    },
+    {
+      "files": ["**/*.{ts,tsx}"],
+      "options": {
+        "semi": false,
+        "singleQuote": true,
+        "printWidth": 100,
+        "jsxBracketSameLine": false,
+        "arrowParens": "always"
+      }
     }
   ]
 }

components/ExampleComponent.tsx

@@ -0,0 +1,3 @@
+export const ExampleComponent = () => {
+  return <div>Welcome to Next.JS land!</div>
+}

feature-flags.json

@@ -1,5 +1,6 @@
 {
   "FEATURE_TEST_TRUE": true,
   "FEATURE_TEST_FALSE": false,
-  "FEATURE_NEW_SITETREE": false
+  "FEATURE_NEW_SITETREE": false,
+  "FEATURE_NEXTJS": false
 }

middleware/csp.js

@@ -35,8 +35,11 @@ module.exports = function csp (req, res, next) {
       ],
       scriptSrc: [
         "'self'",
-        'data:'
-      ],
+        'data:',
+        // For use during development only! This allows us to use a performant webpack devtool setting (eval)
+        // https://webpack.js.org/configuration/devtool/#devtool
+        process.env.NODE_ENV === 'development' && "'unsafe-eval'"
+      ].filter(Boolean),
       frameSrc: [ // exceptions for GraphQL Explorer
         'https://graphql-explorer.githubapp.com', // production env
         'https://graphql.github.com/',

middleware/index.js

@@ -137,6 +137,11 @@ module.exports = function (app) {
   // *** Headers for pages only ***
   app.use(require('./set-fastly-cache-headers'))
 
+  // handle serving NextJS bundled code (/_next/*)
+  if (process.env.FEATURE_NEXTJS) {
+    app.use(instrument('./next'))
+  }
+
   // Check for a dropped connection before proceeding (again)
   app.use(haltOnDroppedConnection)
 

middleware/next.js

@@ -0,0 +1,21 @@
+const next = require('next')
+
+const { NODE_ENV, FEATURE_NEXTJS } = process.env
+const isDevelopment = NODE_ENV === 'development'
+
+let nextHandleRequest
+if (FEATURE_NEXTJS) {
+  const nextApp = next({ dev: isDevelopment })
+  nextHandleRequest = nextApp.getRequestHandler()
+  nextApp.prepare()
+}
+
+module.exports = function renderPageWithNext (req, res, next) {
+  if (req.path.startsWith('/_next/')) {
+    return nextHandleRequest(req, res)
+  }
+
+  next()
+}
+
+module.exports.nextHandleRequest = nextHandleRequest

middleware/render-page.js

@@ -7,8 +7,9 @@ const Page = require('../lib/page')
 const statsd = require('../lib/statsd')
 const RedisAccessor = require('../lib/redis-accessor')
 const { isConnectionDropped } = require('./halt-on-dropped-connection')
+const { nextHandleRequest } = require('./next')
 
-const { HEROKU_RELEASE_VERSION } = process.env
+const { HEROKU_RELEASE_VERSION, FEATURE_NEXTJS } = process.env
 const pageCacheDatabaseNumber = 1
 const pageCacheExpiration = 24 * 60 * 60 * 1000 // 24 hours
 
@@ -25,6 +26,12 @@ const pageCache = new RedisAccessor({
 // a list of query params that *do* alter the rendered page, and therefore should be cached separately
 const cacheableQueries = ['learn']
 
+const renderWithNext = FEATURE_NEXTJS
+  ? [
+      '/en/rest'
+    ]
+  : []
+
 function addCsrf (req, text) {
   return text.replace('$CSRFTOKEN$', req.csrfToken())
 }
@@ -65,7 +72,10 @@ module.exports = async function renderPage (req, res, next) {
   // Is the request for JSON debugging info?
   const isRequestingJsonForDebugging = 'json' in req.query && process.env.NODE_ENV !== 'production'
 
-  if (isCacheable && !isRequestingJsonForDebugging) {
+  // Should the current path be rendered by NextJS?
+  const isNextJsRequest = renderWithNext.includes(req.path)
+
+  if (isCacheable && !isRequestingJsonForDebugging && !(FEATURE_NEXTJS && isNextJsRequest)) {
     // Stop processing if the connection was already dropped
     if (isConnectionDropped(req, res)) return
 
@@ -136,15 +146,19 @@ module.exports = async function renderPage (req, res, next) {
     }
   }
 
-  // currentLayout is added to the context object in middleware/contextualizers/layouts
-  const output = await liquid.parseAndRender(req.context.currentLayout, context)
+  if (FEATURE_NEXTJS && isNextJsRequest) {
+    nextHandleRequest(req, res)
+  } else {
+    // currentLayout is added to the context object in middleware/contextualizers/layouts
+    const output = await liquid.parseAndRender(req.context.currentLayout, context)
 
-  // First, send the response so the user isn't waiting
-  // NOTE: Do NOT `return` here as we still need to cache the response afterward!
-  res.send(addCsrf(req, output))
+    // First, send the response so the user isn't waiting
+    // NOTE: Do NOT `return` here as we still need to cache the response afterward!
+    res.send(addCsrf(req, output))
 
-  // Finally, save output to cache for the next time around
-  if (isCacheable) {
-    await pageCache.set(originalUrl, output, { expireIn: pageCacheExpiration })
+    // Finally, save output to cache for the next time around
+    if (isCacheable) {
+      await pageCache.set(originalUrl, output, { expireIn: pageCacheExpiration })
+    }
   }
 }

next-env.d.ts

@@ -0,0 +1,2 @@
+/// <reference types="next" />
+/// <reference types="next/types/global" />

next.config.js

@@ -0,0 +1,17 @@
+const { productIds } = require('./lib/all-products')
+
+module.exports = {
+  i18n: {
+    locales: ['en', 'ja'],
+    defaultLocale: 'en'
+  },
+  async rewrites () {
+    const defaultVersionId = 'free-pro-team@latest'
+    return productIds.map((productId) => {
+      return {
+        source: `/${productId}/:path*`,
+        destination: `/${defaultVersionId}/${productId}/:path*`
+      }
+    })
+  }
+}

package.json

@@ -71,6 +71,7 @@
     "mini-css-extract-plugin": "^1.4.1",
     "mkdirp": "^1.0.3",
     "morgan": "^1.9.1",
+    "next": "^10.2.0",
     "node-fetch": "^2.6.1",
     "parse5": "^6.0.1",
     "port-used": "^2.0.8",
@@ -111,6 +112,8 @@
     "@actions/core": "^1.2.6",
     "@actions/github": "^4.0.0",
     "@octokit/rest": "^16.43.2",
+    "@types/react": "^17.0.4",
+    "@types/react-dom": "^17.0.3",
     "async": "^3.2.0",
     "await-sleep": "0.0.1",
     "aws-sdk": "^2.610.0",
@@ -163,6 +166,7 @@
     "start-server-and-test": "^1.12.0",
     "strip-ansi": "^6.0.0",
     "supertest": "^4.0.2",
+    "typescript": "^4.2.4",
     "url-template": "^2.0.8",
     "webpack-dev-middleware": "^4.1.0",
     "website-scraper": "^4.2.0",
@@ -175,8 +179,9 @@
     "rest-dev": "script/rest/update-files.js && npm run dev",
     "build": "cross-env NODE_ENV=production npx webpack --mode production",
     "start-all-languages": "cross-env NODE_ENV=development nodemon server.js",
-    "lint": "eslint --fix . && prettier -w \"**/*.{yml,yaml}\"",
+    "lint": "eslint --fix . && prettier -w \"**/*.{yml,yaml}\" && npm run lint-tsc",
     "lint-translation": "TEST_TRANSLATION=true jest content/lint-files",
+    "lint-tsc": "prettier -w \"**/*.{ts,tsx}\"",
     "test": "jest && eslint . && prettier -c \"**/*.{yml,yaml}\" && npm run check-deps",
     "prebrowser-test": "npm run build",
     "browser-test": "start-server-and-test browser-test-server 4001 browser-test-tests",

pages/[versionId]/[categoryId]/index.tsx

@@ -0,0 +1,12 @@
+import { ExampleComponent } from 'components/ExampleComponent'
+
+const CategoryPage = () => {
+  return (
+    <div className="p-4">
+      <h1>Sample category page</h1>
+      <ExampleComponent />
+    </div>
+  )
+}
+
+export default CategoryPage

pages/_app.tsx

@@ -0,0 +1,34 @@
+import React from 'react'
+import { AppProps } from 'next/app'
+import Head from 'next/head'
+
+import '@primer/css/index.scss'
+
+const App: React.FC<AppProps> = ({ Component, pageProps }) => {
+  return (
+    <>
+      <Head>
+        <meta charSet="utf-8" />
+        <title>GitHub Documentation</title>
+        <meta name="viewport" content="width=device-width, initial-scale=1" />
+
+        <link rel="alternate icon" type="image/png" href="/assets/images/site/favicon.png" />
+        <link rel="icon" type="image/svg+xml" href="/assets/images/site/favicon.svg" />
+
+        <meta
+          name="google-site-verification"
+          content="OgdQc0GZfjDI52wDv1bkMT-SLpBUo_h5nn9mI9L22xQ"
+        />
+        <meta
+          name="google-site-verification"
+          content="c1kuD-K2HIVF635lypcsWPoD4kilo5-jA_wBFyT4uMY"
+        />
+
+        <meta name="csrf-token" content="$CSRFTOKEN$" />
+      </Head>
+      <Component {...pageProps} />
+    </>
+  )
+}
+
+export default App

tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noImplicitThis": false,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "baseUrl": ".",
+  },
+  "exclude": ["node_modules"],
+  "include": ["*.d.ts", "**/*.ts", "**/*.tsx"]
+}

webpack.config.js

@@ -6,7 +6,7 @@ const { reactBabelOptions } = require('./lib/react/babel')
 
 module.exports = {
   mode: 'development',
-  devtool: 'source-map', // this prevents webpack from using eval
+  devtool: process.env.NODE_ENV === 'development' ? 'eval' : 'source-map', // no 'eval' outside of development
   entry: './javascripts/index.js',
   output: {
     filename: 'index.js',