This commit is contained in:
James M. Greene 2020-11-09 13:07:04 -06:00
Родитель ed3baeb5dc
Коммит f410fd175c
19 изменённых файлов: 136 добавлений и 283 удалений

Просмотреть файл

@ -1,6 +1,3 @@
ALGOLIA_API_KEY=
ALGOLIA_APPLICATION_ID=
ALLOW_TRANSLATION_COMMITS=
EARLY_ACCESS_HOSTNAME=
EARLY_ACCESS_SHARED_SECRET=
GITHUB_TOKEN=

7
.github/workflows/test.yml поставляемый
Просмотреть файл

@ -127,10 +127,3 @@ jobs:
run: npx jest tests/${{ matrix.test-group }}/
env:
NODE_OPTIONS: "--max_old_space_size=4096"
- name: Send Slack notification if workflow fails
uses: rtCamp/action-slack-notify@e17352feaf9aee300bf0ebc1dfbf467d80438815
if: failure() && github.ref == 'early-access'
env:
SLACK_WEBHOOK: ${{ secrets.DOCS_ALERTS_SLACK_WEBHOOK }}
SLACK_MESSAGE: "Tests are failing on the `early-access` branch. https://github.com/github/docs-internal/tree/early-access"

11
.gitignore поставляемый
Просмотреть файл

@ -1,13 +1,14 @@
.algolia-cache
.DS_Store
.env
node_modules
/node_modules/
npm-debug.log
coverage
content/early-access
content/early-access-test
coverage/
/assets/early-access/
/content/early-access/
/data/early-access/
# blc: broken link checker
blc_output.log
blc_output_internal.log
dist
/dist/

Просмотреть файл

@ -14,9 +14,10 @@ files:
"data/reusables/README.md",
"data/variables/product.yml",
"data/variables/README.md",
"data/early-access",
"data/graphql",
"data/products.yml"
]
]
# These end up as env vars used by the GitHub Actions workflow
project_id_env: CROWDIN_PROJECT_ID

Просмотреть файл

@ -1,33 +0,0 @@
// This module loads an array of Early Access page paths from EARLY_ACCESS_HOSTNAME
//
// See also middleware/early-acces-proxy.js which fetches Early Access docs from the obscured remote host
require('dotenv').config()
const got = require('got')
const isURL = require('is-url')
module.exports = async function fetchEarlyAccessPaths () {
let url
if (process.env.NODE_ENV === 'test') return []
if (!isURL(process.env.EARLY_ACCESS_HOSTNAME)) {
console.log('EARLY_ACCESS_HOSTNAME is not defined; skipping fetching early access paths')
return []
}
try {
url = `${process.env.EARLY_ACCESS_HOSTNAME}/early-access-paths.json`
const { body } = await got(url, {
json: true,
timeout: 3000,
headers: {
'early-access-shared-secret': process.env.EARLY_ACCESS_SHARED_SECRET
}
})
return body
} catch (err) {
console.error('Unable to fetch early-access-paths.json from', url, err)
return []
}
}

Просмотреть файл

@ -1,5 +1,4 @@
const fetchEarlyAccessPaths = require('./fetch-early-access-paths')
let pages, site, redirects, siteTree, earlyAccessPaths
let pages, site, redirects, siteTree
module.exports = async function warmServer () {
if (!pages) {
@ -8,10 +7,9 @@ module.exports = async function warmServer () {
}
// Promise.all is used to load multiple things in parallel
;[pages, site, earlyAccessPaths] = await Promise.all([
;[pages, site] = await Promise.all([
require('./pages')(),
require('./site-data')(),
fetchEarlyAccessPaths()
require('./site-data')()
])
redirects = await require('./redirects/precompile')(pages)
@ -19,6 +17,6 @@ module.exports = async function warmServer () {
}
return {
pages, site, redirects, siteTree, earlyAccessPaths
pages, site, redirects, siteTree
}
}

Просмотреть файл

@ -12,7 +12,7 @@ const featureFlags = Object.keys(require('../feature-flags'))
// Note that additional middleware in middleware/index.js adds to this context object
module.exports = async function contextualize (req, res, next) {
// Ensure that we load some data only once on first request
const { site, redirects, pages, siteTree, earlyAccessPaths } = await warmServer()
const { site, redirects, pages, siteTree } = await warmServer()
req.context = {}
// make feature flag environment variables accessible in layouts
@ -33,7 +33,6 @@ module.exports = async function contextualize (req, res, next) {
req.context.currentPath = req.path
req.context.query = req.query
req.context.languages = languages
req.context.earlyAccessPaths = earlyAccessPaths
req.context.productNames = productNames
req.context.enterpriseServerReleases = enterpriseServerReleases
req.context.enterpriseServerVersions = Object.keys(allVersions).filter(version => version.startsWith('enterprise-server@'))

Просмотреть файл

@ -1,33 +0,0 @@
const { chain } = require('lodash')
let paths
// This middleware finds all pages with `hidden: true` frontmatter
// and responds with a JSON array of all requests paths (and redirects) that lead to those pages.
// Requesting this path from EARLY_ACCESS_HOSTNAME will respond with an array of Early Access paths.
// Requesting this path from docs.github.com (production) will respond with an empty array (no Early Access paths).
module.exports = async (req, res, next) => {
if (req.path !== '/early-access-paths.json') return next()
if (
!req.headers ||
!req.headers['early-access-shared-secret'] ||
req.headers['early-access-shared-secret'] !== process.env.EARLY_ACCESS_SHARED_SECRET
) {
return res.status(401).send({ error: '401 Unauthorized' })
}
paths = paths || chain(req.context.pages)
.filter(page => page.hidden && page.languageCode === 'en')
.map(page => {
const permalinks = page.permalinks.map(permalink => permalink.href)
const redirects = Object.keys(page.redirects)
return permalinks.concat(redirects)
})
.flatten()
.uniq()
.value()
return res.json(paths)
}

Просмотреть файл

@ -1,25 +0,0 @@
// This module serves requests to Early Access content from a hidden proxy host (EARLY_ACCESS_HOSTNAME).
// Paths to this content are fetched in the warmServer module at startup.
const got = require('got')
const isURL = require('is-url')
module.exports = async (req, res, next) => {
if (
isURL(process.env.EARLY_ACCESS_HOSTNAME) &&
req.context &&
req.context.earlyAccessPaths &&
req.context.earlyAccessPaths.includes(req.path)
) {
try {
const proxyURL = `${process.env.EARLY_ACCESS_HOSTNAME}${req.path}`
const proxiedRes = await got(proxyURL)
res.set('content-type', proxiedRes.headers['content-type'])
res.send(proxiedRes.body)
} catch (err) {
next()
}
} else {
next()
}
}

Просмотреть файл

@ -40,8 +40,6 @@ module.exports = function (app) {
app.use(require('./detect-language'))
app.use(asyncMiddleware(require('./context')))
app.use('/csrf', require('./csrf-route'))
app.use(require('./early-access-paths'))
app.use(require('./early-access-proxy'))
app.use(require('./find-page'))
app.use(require('./notices'))
app.use(require('./archived-enterprise-versions'))
@ -56,6 +54,7 @@ module.exports = function (app) {
app.use(require('./contextualizers/webhooks'))
app.use(require('./disable-caching-on-safari'))
app.get('/_500', asyncMiddleware(require('./trigger-error')))
app.get('/hidden', require('./list-hidden-pages'))
app.use(require('./breadcrumbs'))
app.use(require('./featured-links'))
app.get('/*', asyncMiddleware(require('./render-page')))

Просмотреть файл

@ -0,0 +1,21 @@
module.exports = async function listHidden (req, res, next) {
if (process.env.NODE_ENV === 'production') {
return res.status(403).end()
}
const hiddenPages = req.context.pages.filter(page => page.hidden)
let urls = []
hiddenPages.forEach(page => {
const pageUrls = page.permalinks.map(permalink => permalink.href)
urls = urls.concat(pageUrls)
})
const output = `
<ul>
${urls.map(url => `<li><a href="${url}">${url}</li>`).join('\n')}
</ul>
`
return res.send(output)
}

Просмотреть файл

@ -1,5 +1,4 @@
const { get } = require('lodash')
const env = require('lil-env-thing')
const { liquid } = require('../lib/render-content')
const patterns = require('../lib/patterns')
const layouts = require('../lib/layouts')
@ -64,7 +63,7 @@ module.exports = async function renderPage (req, res, next) {
}
// `?json` query param for debugging request context
if ('json' in req.query && !env.production) {
if ('json' in req.query && process.env.NODE_ENV !== 'production') {
if (req.query.json.length > 1) {
// deep reference: ?json=page.permalinks
return res.json(get(context, req.query.json))

29
package-lock.json сгенерированный
Просмотреть файл

@ -3169,7 +3169,7 @@
},
"agentkeepalive": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-2.2.0.tgz",
"resolved": "http://registry.npmjs.org/agentkeepalive/-/agentkeepalive-2.2.0.tgz",
"integrity": "sha1-xdG9SxKQCPEWPyNvhuX66iAm4u8="
},
"aggregate-error": {
@ -3329,7 +3329,7 @@
"argparse": {
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
"integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
"integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=",
"requires": {
"sprintf-js": "~1.0.2"
}
@ -4114,7 +4114,7 @@
},
"brfs": {
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/brfs/-/brfs-1.6.1.tgz",
"resolved": "http://registry.npmjs.org/brfs/-/brfs-1.6.1.tgz",
"integrity": "sha512-OfZpABRQQf+Xsmju8XE9bDjs+uU4vLREGolP7bDgcpsI17QREyZ4Bl+2KLxxx1kCgA0fAIhKQBaBYh+PEcCqYQ==",
"requires": {
"quote-stream": "^1.0.1",
@ -4193,7 +4193,7 @@
},
"browserify-aes": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz",
"resolved": "http://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz",
"integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==",
"requires": {
"buffer-xor": "^1.0.3",
@ -4227,7 +4227,7 @@
},
"browserify-rsa": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
"resolved": "http://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
"integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=",
"requires": {
"bn.js": "^4.1.0",
@ -5645,7 +5645,7 @@
},
"create-hash": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
"resolved": "http://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
"integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==",
"requires": {
"cipher-base": "^1.0.1",
@ -5657,7 +5657,7 @@
},
"create-hmac": {
"version": "1.1.7",
"resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz",
"resolved": "http://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz",
"integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==",
"requires": {
"cipher-base": "^1.0.3",
@ -6384,7 +6384,7 @@
},
"diffie-hellman": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
"resolved": "http://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
"integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==",
"requires": {
"bn.js": "^4.1.0",
@ -6824,7 +6824,7 @@
"error-ex": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
"integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==",
"integrity": "sha1-tKxAZIEH/c3PriQvQovqihTU8b8=",
"dev": true,
"requires": {
"is-arrayish": "^0.2.1"
@ -9962,7 +9962,7 @@
"dependencies": {
"mkdirp": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz",
"resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz",
"integrity": "sha1-G79asbqCevI1dRQ0kEJkVfSB/h4="
},
"nopt": {
@ -14899,11 +14899,6 @@
"type-check": "~0.3.2"
}
},
"lil-env-thing": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/lil-env-thing/-/lil-env-thing-1.0.0.tgz",
"integrity": "sha1-etQmBiG/M1rR6HE1d5s15vFmxns="
},
"limited-request-queue": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/limited-request-queue/-/limited-request-queue-2.0.0.tgz",
@ -15143,7 +15138,7 @@
},
"magic-string": {
"version": "0.22.5",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz",
"resolved": "http://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz",
"integrity": "sha512-oreip9rJZkzvA8Qzk9HFs8fZGF/u7H/gtrE8EN6RjKJ9kh2HlC+yQ2QezifqTZfGyiuAV0dRv5a+y/8gBb1m9w==",
"requires": {
"vlq": "^0.2.2"
@ -18704,7 +18699,7 @@
},
"sha.js": {
"version": "2.4.11",
"resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz",
"resolved": "http://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz",
"integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==",
"requires": {
"inherits": "^2.0.1",

Просмотреть файл

@ -49,7 +49,6 @@
"is-url": "^1.2.4",
"js-cookie": "^2.2.1",
"js-yaml": "^3.14.0",
"lil-env-thing": "^1.0.0",
"liquid": "^5.1.0",
"lodash": "^4.17.19",
"mini-css-extract-plugin": "^0.9.0",

Просмотреть файл

@ -1,9 +1,46 @@
const config = require('../../lib/crowdin-config').read()
const loadPages = require('../../lib/pages')
const ignoredPagePaths = config.files[0].ignore
const ignoredDataPaths = config.files[2].ignore
describe('crowdin.yml config file', () => {
let pages
beforeAll(async (done) => {
pages = await loadPages()
done()
})
test('has expected file stucture', async () => {
expect(config.files.length).toBe(3)
expect(config.files[0].source).toBe('/content/**/*.md')
expect(config.files[0].ignore).toContain('/content/README.md')
})
test('ignores all Early Access paths', async () => {
expect(ignoredPagePaths).toContain('content/early-access')
expect(ignoredDataPaths).toContain('data/early-access')
})
test('ignores all hidden pages', async () => {
const hiddenPages = pages
.filter(page => page.hidden && page.languageCode === 'en')
.map(page => `/content/${page.relativePath}`)
const overlooked = hiddenPages.filter(page => !isIgnored(page, ignoredPagePaths))
const message = `Found some hidden pages that are not yet excluded from localization.
Please copy and paste the lines below into the \`ignore\` section of /crowdin.yml: \n\n"${overlooked.join('",\n"')}"`
// This may not be true anymore given the separation of Early Access docs
// expect(hiddenPages.length).toBeGreaterThan(0)
expect(ignoredPagePaths.length).toBeGreaterThan(0)
expect(overlooked, message).toHaveLength(0)
})
})
// file is ignored if its exact filename in the list,
// or if it's within an ignored directory
function isIgnored (filename, ignoredPagePaths) {
return ignoredPagePaths.some(ignoredPath => {
const isDirectory = !ignoredPath.endsWith('.md')
return ignoredPath === filename || (isDirectory && filename.startsWith(ignoredPath))
})
}

Просмотреть файл

@ -1,64 +0,0 @@
const MockExpressResponse = require('mock-express-response')
const middleware = require('../../middleware/early-access-paths')
describe('GET /early-access-paths.json', () => {
beforeEach(() => {
delete process.env['early-access-shared-secret']
})
test('responds with 401 if shared secret is missing', async () => {
const req = {
path: '/early-access-paths.json',
headers: {}
}
const res = new MockExpressResponse()
const next = jest.fn()
await middleware(req, res, next)
expect(res._getJSON()).toEqual({ error: '401 Unauthorized' })
})
test('responds with an array of hidden paths', async () => {
process.env.EARLY_ACCESS_SHARED_SECRET = 'bananas'
const req = {
path: '/early-access-paths.json',
headers: {
'early-access-shared-secret': 'bananas'
},
context: {
pages: [
{
hidden: true,
languageCode: 'en',
permalinks: [
{ href: '/some-hidden-page' }
],
redirects: {
'/old-hidden-page': '/new-hidden-page'
}
},
{
hidden: false,
languageCode: 'en'
}
]
}
}
const res = new MockExpressResponse()
const next = jest.fn()
await middleware(req, res, next)
expect(res._getJSON()).toEqual(['/some-hidden-page', '/old-hidden-page'])
})
test('ignores requests to other paths', async () => {
const req = {
path: '/not-early-access'
}
const res = new MockExpressResponse()
const next = jest.fn()
await middleware(req, res, next)
expect(next).toHaveBeenCalled()
})
})

Просмотреть файл

@ -1,80 +0,0 @@
const middleware = require('../../middleware/early-access-proxy')
const nock = require('nock')
const MockExpressResponse = require('mock-express-response')
describe('Early Access middleware', () => {
const OLD_EARLY_ACCESS_HOSTNAME = process.env.EARLY_ACCESS_HOSTNAME
beforeAll(() => {
process.env.EARLY_ACCESS_HOSTNAME = 'https://secret-website.com'
})
afterAll(() => {
process.env.EARLY_ACCESS_HOSTNAME = OLD_EARLY_ACCESS_HOSTNAME
})
const baseReq = {
context: {
earlyAccessPaths: ['/alpha-product/foo', '/beta-product/bar', '/baz']
}
}
test('are proxied from an obscured host', async () => {
const mock = nock('https://secret-website.com')
.get('/alpha-product/foo')
.reply(200, 'yay here is your proxied content', { 'content-type': 'text/html' })
const req = { ...baseReq, path: '/alpha-product/foo' }
const res = new MockExpressResponse()
const next = jest.fn()
await middleware(req, res, next)
expect(mock.isDone()).toBe(true)
expect(res._getString()).toBe('yay here is your proxied content')
})
test('follows redirects', async () => {
const mock = nock('https://secret-website.com')
.get('/alpha-product/foo')
.reply(301, undefined, { Location: 'https://secret-website.com/alpha-product/foo2' })
.get('/alpha-product/foo2')
.reply(200, 'yay you survived the redirect', { 'content-type': 'text/html' })
const req = { ...baseReq, path: '/alpha-product/foo' }
const res = new MockExpressResponse()
const next = jest.fn()
await middleware(req, res, next)
expect(mock.isDone()).toBe(true)
expect(res._getString()).toBe('yay you survived the redirect')
})
test('calls next() if no redirect is found', async () => {
const req = { ...baseReq, path: '/en' }
const res = new MockExpressResponse()
const next = jest.fn()
await middleware(req, res, next)
expect(next).toHaveBeenCalled()
})
test('calls next() if proxy request respond with 404', async () => {
const mock = nock('https://secret-website.com')
.get('/beta-product/bar')
.reply(404, 'no dice', { 'content-type': 'text/html' })
const req = { ...baseReq, path: '/beta-product/bar' }
const res = new MockExpressResponse()
const next = jest.fn()
await middleware(req, res, next)
expect(mock.isDone()).toBe(true)
expect(next).toHaveBeenCalled()
})
test('calls next() if proxy request responds with 500', async () => {
const mock = nock('https://secret-website.com')
.get('/beta-product/bar')
.reply(500, 'no dice', { 'content-type': 'text/html' })
const req = { ...baseReq, path: '/beta-product/bar' }
const res = new MockExpressResponse()
const next = jest.fn()
await middleware(req, res, next)
expect(mock.isDone()).toBe(true)
expect(next).toHaveBeenCalled()
})
})

Просмотреть файл

@ -3,6 +3,7 @@ const enterpriseServerReleases = require('../../lib/enterprise-server-releases')
const { get, getDOM, head } = require('../helpers')
const path = require('path')
const nonEnterpriseDefaultVersion = require('../../lib/non-enterprise-default-version')
const loadPages = require('../../lib/pages')
describe('server', () => {
jest.setTimeout(60 * 1000)
@ -375,6 +376,44 @@ describe('server', () => {
})
})
describe('hidden articles', () => {
let hiddenPageHrefs, hiddenPages
beforeAll(async (done) => {
const $ = await getDOM('/hidden')
hiddenPageHrefs = $('a').map((i, el) => $(el).attr('href')).get()
const allPages = await loadPages()
hiddenPages = allPages.filter(page => page.languageCode === 'en' && page.hidden)
done()
})
test('are listed at /hidden', async () => {
expect(hiddenPageHrefs.length).toBe(hiddenPages.length)
})
test('are not listed at /hidden in production', async () => {
const oldNodeEnv = process.env.NODE_ENV
process.env.NODE_ENV = 'production'
const res = await get('/hidden')
process.env.NODE_ENV = oldNodeEnv
expect(res.statusCode).toBe(403)
})
test('have noindex meta tags', async () => {
if (hiddenPageHrefs.length > 0) {
const $ = await getDOM(hiddenPageHrefs[0])
expect($('meta[content="noindex"]').length).toBe(1)
}
})
test('non-hidden articles do not have noindex meta tags', async () => {
const $ = await getDOM('/en/articles/set-up-git')
expect($('meta[content="noindex"]').length).toBe(0)
})
})
describe('redirects', () => {
test('redirects old articles to their English URL', async () => {
const res = await get('/articles/deleting-a-team')

Просмотреть файл

@ -6,9 +6,19 @@ const testViaActionsOnly = process.env.GITHUB_ACTIONS ? test : test.skip
// TODO test ea-config
// TODO this should only run locally
describe('cloning content/early-access', () => {
describe('cloning early-access', () => {
testViaActionsOnly('the assets directory exists', async () => {
const eaContentDir = path.join(process.cwd(), 'assets/early-access')
expect(fs.existsSync(eaContentDir)).toBe(true)
})
testViaActionsOnly('the content directory exists', async () => {
const eaContentDir = path.join(process.cwd(), 'content/early-access-test')
const eaContentDir = path.join(process.cwd(), 'content/early-access')
expect(fs.existsSync(eaContentDir)).toBe(true)
})
testViaActionsOnly('the data directory exists', async () => {
const eaContentDir = path.join(process.cwd(), 'data/early-access')
expect(fs.existsSync(eaContentDir)).toBe(true)
})
})