зеркало из https://github.com/github/docs.git
Merge branch 'main' into revert-16952-revert-16947-optimize-sitetree
This commit is contained in:
Sarah Schneider
GitHub
Коммит
969c8d5110
3
app.json
3
app.json
|
|
@ -3,10 +3,9 @@
|
|||
"env": {
|
||||
"NODE_ENV": "production",
|
||||
"NPM_CONFIG_PRODUCTION": "true",
|
||||
"ENABLED_LANGUAGES": "en, de"
|
||||
"ENABLED_LANGUAGES": "en"
|
||||
},
|
||||
"buildpacks": [
|
||||
{ "url": "https://github.com/DataDog/heroku-buildpack-datadog.git#1.21" },
|
||||
{ "url": "heroku/nodejs" }
|
||||
],
|
||||
"formation": {
|
||||
|
|
|
|||
|
|
@ -79,8 +79,8 @@ Permission | Description
|
|||
[`single_file`](/rest/reference/permissions-required-for-github-apps/#permission-on-single-file) | Grants access to the [Contents API](/rest/reference/repos#contents). Can be one of: `none`, `read`, or `write`.
|
||||
[`starring`](/rest/reference/permissions-required-for-github-apps/#permission-on-starring) | Grants access to the [Starring API](/rest/reference/activity#starring). Can be one of: `none`, `read`, or `write`.
|
||||
[`statuses`](/rest/reference/permissions-required-for-github-apps/#permission-on-statuses) | Grants access to the [Statuses API](/rest/reference/repos#statuses). Can be one of: `none`, `read`, or `write`.
|
||||
[`team_discussions`](/rest/reference/permissions-required-for-github-apps/#permission-on-team-discussions) | Grants access to the [Team Discussions API](/rest/reference/teams#discussions) and the [Team Discussion Comments API](/rest/reference/teams#discussion-comments). Can be one of: `none`, `read`, or `write`.
|
||||
`vulnerability_alerts`| Grants access to receive security alerts for vulnerable dependencies in a repository. See "[About security alerts for vulnerable dependencies](/articles/about-security-alerts-for-vulnerable-dependencies)" to learn more. Can be one of: `none` or `read`.
|
||||
[`team_discussions`](/rest/reference/permissions-required-for-github-apps/#permission-on-team-discussions) | Grants access to the [Team Discussions API](/rest/reference/teams#discussions) and the [Team Discussion Comments API](/rest/reference/teams#discussion-comments). Can be one of: `none`, `read`, or `write`.{% if currentVersion == "free-pro-team@latest" or currentVersion ver_gt "enterprise-server@1.19" %}
|
||||
`vulnerability_alerts`| Grants access to receive security alerts for vulnerable dependencies in a repository. See "[About security alerts for vulnerable dependencies](/articles/about-security-alerts-for-vulnerable-dependencies)" to learn more. Can be one of: `none` or `read`.{% endif %}
|
||||
`watching` | Grants access to list and change repositories a user is subscribed to. Can be one of: `none`, `read`, or `write`.
|
||||
|
||||
### {% data variables.product.prodname_github_app %} webhook events
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ If the user accepts your request, GitHub redirects back to your site with a temp
|
|||
|
||||
{% endnote %}
|
||||
|
||||
Exchange this `code` for an access token. {% if currentVersion == "free-pro-team@latest" or currentVersion ver_gt "enterprise-server@2.21" or currentVersion == "github-ae@latest" %} When expiring tokens are enabled, the access token expires in 8 hours and the refresh token expires in 6 months. Every time you refresh the token, you get a new refresh token. For more information, see "[Refreshing user-to-server access tokens](/developers/apps/refreshing-user-to-server-access-tokens)."
|
||||
Exchange this `code` for an access token. {% if currentVersion == "free-pro-team@latest" %} When expiring tokens are enabled, the access token expires in 8 hours and the refresh token expires in 6 months. Every time you refresh the token, you get a new refresh token. For more information, see "[Refreshing user-to-server access tokens](/developers/apps/refreshing-user-to-server-access-tokens)."
|
||||
|
||||
Expiring user tokens are currently part of the user-to-server token expiration beta and subject to change. To opt-in to the user-to-server token expiration beta feature, see "[Activating beta features for apps](/developers/apps/activating-beta-features-for-apps)."{% endif %}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,13 +7,11 @@ versions:
|
|||
free-pro-team: '*'
|
||||
---
|
||||
|
||||
{% if currentVersion == "free-pro-team@latest" or currentVersion ver_gt "enterprise-server@2.21" %}
|
||||
{% note %}
|
||||
|
||||
**Note:** {% data reusables.pre-release-program.suspend-installation-beta %}
|
||||
|
||||
{% endnote %}
|
||||
{% endif %}
|
||||
|
||||
### Suspending a GitHub App
|
||||
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ Customers can start a free trial for any paid plan on a Marketplace listing that
|
|||
|
||||
Free trials have a fixed length of 14 days. Customers are notified 4 days before the end of their trial period (on day 11 of the free trial) that their plan will be upgraded. At the end of a free trial, customers will be auto-enrolled into the plan they are trialing if they do not cancel.
|
||||
|
||||
For more information, see: "[Handling new purchases and free trials](/developers/github-marketplace/integrating-with-the-github-marketplace-api/handling-new-purchases-and-free-trials/)."
|
||||
For more information, see: "[Handling new purchases and free trials](/developers/github-marketplace/handling-new-purchases-and-free-trials/)."
|
||||
|
||||
{% note %}
|
||||
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ In addition to the requirements for all apps above, each app that you offer as a
|
|||
- {% data variables.product.prodname_github_app %}s should have a minimum of 100 installations.
|
||||
- {% data variables.product.prodname_oauth_app %}s should have a minimum of 200 users.
|
||||
- All paid apps must handle {% data variables.product.prodname_marketplace %} purchase events for new purchases, upgrades, downgrades, cancellations, and free trials. For more information, see "[Billing requirements for paid apps](#billing-requirements-for-paid-apps)" below.
|
||||
- Publishing organizations must have a verified domain and must enable two-factor authentication. For more information, see "[Requiring two-factor authentication in your organization](/github/setting-up-and-managing-organizations-and-teams/requiring-two-factor-authentication-in-your-organization.")
|
||||
- Publishing organizations must have a verified domain and must enable two-factor authentication. For more information, see "[Requiring two-factor authentication in your organization](/github/setting-up-and-managing-organizations-and-teams/requiring-two-factor-authentication-in-your-organization)."
|
||||
|
||||
When you are ready to publish the app on {% data variables.product.prodname_marketplace %} you must request verification for the listing.
|
||||
|
||||
|
|
|
|||
|
|
@ -44,7 +44,9 @@ If you don't want to use SSH keys, you can use [HTTPS with OAuth tokens][git-aut
|
|||
* Users don't have to change their local SSH settings.
|
||||
* Multiple tokens (one for each user) are not needed; one token per server is enough.
|
||||
* A token can be revoked at any time, turning it essentially into a one-use password.
|
||||
{% if enterpriseServerVersions contains currentVersion %}
|
||||
* Generating new tokens can be easily scripted using [the OAuth API](/rest/reference/oauth-authorizations#create-a-new-authorization).
|
||||
{% endif %}
|
||||
|
||||
##### Cons
|
||||
|
||||
|
|
|
|||
|
|
@ -430,7 +430,7 @@ Key | Type | Description
|
|||
|
||||
{% endnote %}
|
||||
|
||||
{% if currentVersion == "free-pro-team@latest" or currentVersion ver_gt "enterprise-server@2.21" or currentVersion == "github-ae@latest" %}
|
||||
{% if currentVersion == "free-pro-team@latest" %}
|
||||
{% note %}
|
||||
|
||||
**Note:** {% data reusables.pre-release-program.suspend-installation-beta %} For more information, see "[Suspending a {% data variables.product.prodname_github_app %} installation](/apps/managing-github-apps/suspending-a-github-app-installation/)."
|
||||
|
|
@ -1124,9 +1124,11 @@ Key | Type | Description
|
|||
{{ webhookPayloadsForCurrentVersion.secret_scanning_alert.reopened }}
|
||||
{% endif %}
|
||||
|
||||
{% if currentVersion == "free-pro-team@latest" or currentVersion ver_gt "enterprise-server@1.19" %}
|
||||
### security_advisory
|
||||
|
||||
Activity related to a security advisory. A security advisory provides information about security-related vulnerabilities in software on GitHub. The security advisory dataset also powers the GitHub security alerts, see "[About security alerts for vulnerable dependencies](/articles/about-security-alerts-for-vulnerable-dependencies/)."
|
||||
{% endif %}
|
||||
|
||||
#### Availability
|
||||
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ When an {% data variables.product.prodname_oauth_app %} wants to identify you by
|
|||
|
||||
*Scopes* are named groups of permissions that an {% data variables.product.prodname_oauth_app %} can request to access both public and non-public data.
|
||||
|
||||
When you want to use an {% data variables.product.prodname_oauth_app %} that integrates with {% data variables.product.product_name %}, that app lets you know what type of access to your data will be required. If you grant access to the app, then the app will be able to perform actions on your behalf, such as reading or modifying data. For example, if you want to use an app that requests `user:email` scope, the app will have read-only access to your private email addresses. For more information, see "[About scopes for {% data variables.product.prodname_oauth_app %}s](//apps/building-integrations/setting-up-and-registering-oauth-apps/about-scopes-for-oauth-apps)."
|
||||
When you want to use an {% data variables.product.prodname_oauth_app %} that integrates with {% data variables.product.product_name %}, that app lets you know what type of access to your data will be required. If you grant access to the app, then the app will be able to perform actions on your behalf, such as reading or modifying data. For example, if you want to use an app that requests `user:email` scope, the app will have read-only access to your private email addresses. For more information, see "[About scopes for {% data variables.product.prodname_oauth_app %}s](/apps/building-integrations/setting-up-and-registering-oauth-apps/about-scopes-for-oauth-apps)."
|
||||
|
||||
{% tip %}
|
||||
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ Applications can have *read* or *write* access to your {% data variables.product
|
|||
|
||||
*Scopes* are named groups of permissions that an application can request to access both public and non-public data.
|
||||
|
||||
When you want to use a third-party application that integrates with {% data variables.product.product_name %}, that application lets you know what type of access to your data will be required. If you grant access to the application, then the application will be able to perform actions on your behalf, such as reading or modifying data. For example, if you want to use an app that requests `user:email` scope, the app will have read-only access to your private email addresses. For more information, see "[About scopes for {% data variables.product.prodname_oauth_app %}s](//apps/building-integrations/setting-up-and-registering-oauth-apps/about-scopes-for-oauth-apps)."
|
||||
When you want to use a third-party application that integrates with {% data variables.product.product_name %}, that application lets you know what type of access to your data will be required. If you grant access to the application, then the application will be able to perform actions on your behalf, such as reading or modifying data. For example, if you want to use an app that requests `user:email` scope, the app will have read-only access to your private email addresses. For more information, see "[About scopes for {% data variables.product.prodname_oauth_app %}s](/apps/building-integrations/setting-up-and-registering-oauth-apps/about-scopes-for-oauth-apps)."
|
||||
|
||||
{% tip %}
|
||||
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ On Windows, the `codeql-runner-win.exe` file usually requires no change to permi
|
|||
Once you have downloaded the {% data variables.product.prodname_codeql_runner %} and verified that it can be executed, you should make the runner available to each CI server that you intend to use for {% data variables.product.prodname_code_scanning %}. It is important to notice that each CI server that you intend to use for {% data variables.product.prodname_code_scanning %} needs to have the {% data variables.product.prodname_codeql_runner %}. You might configure each server to copy the runner from a central, internal location, or you could use the REST API to get the runner direct from GitHub, for example:
|
||||
|
||||
```shell
|
||||
wget https://github.com/github/codeql-action/releases/download/codeql-bundle-20200826/codeql-runner-linux
|
||||
wget https://github.com/github/codeql-action/releases/latest/download/codeql-runner-linux
|
||||
chmod +x codeql-runner-linux
|
||||
```
|
||||
|
||||
|
|
@ -127,7 +127,7 @@ This example is similar to the previous example, however this time the repositor
|
|||
> ...
|
||||
> CodeQL environment output to "/srv/checkout/example-repo-2/codeql-runner/codeql-env.json"
|
||||
and "/srv/checkout/example-repo-2/codeql-runner/codeql-env.sh".
|
||||
Please export these variables to future processes so the build can be traced, for example by running "
|
||||
Please export these variables to future processes so that CodeQL can monitor the build, for example by running "
|
||||
. /srv/checkout/example-repo-2/codeql-runner/codeql-env.sh".
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ To help you understand your subscriptions and decide whether to unsubscribe, see
|
|||
|
||||
{% note %}
|
||||
|
||||
**Note:** Instead of unsubscribing, you have the option to ignore a repository. If you ignore a repository, you won't receive any notifications. We don't recommend ignoring repositories as you won't be notified if you're @mentioned. {% if currentVersion == "free-pro-team@latest" %}If you're experiencing abuse and want to ignore a repository, please [contact support](/contact) so we can help. {% data reusables.policies.abuse %}{% endif %}
|
||||
**Note:** Instead of unsubscribing, you have the option to ignore a repository. If you ignore a repository, you won't receive any notifications. We don't recommend ignoring repositories as you won't be notified if you're @mentioned. {% if currentVersion == "free-pro-team@latest" %}If you're experiencing abuse and want to ignore a repository, please contact {% data variables.contact.contact_support %} so we can help. {% data reusables.policies.abuse %}{% endif %}
|
||||
|
||||
{% endnote %}
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ You can also watch and unwatch releases in a repository. For more information, s
|
|||
|
||||
{% note %}
|
||||
|
||||
**Note:** You can also choose to ignore a repository. If you ignore a repository, you won't receive any notifications. We don't recommend ignoring repositories as you won't be notified if you're @mentioned. {% if currentVersion == "free-pro-team@latest" %}If you experiencing abuse and want to ignore a repository, please [contact support](/contact) so we can help. {% data reusables.policies.abuse %}{% endif %}
|
||||
**Note:** You can also choose to ignore a repository. If you ignore a repository, you won't receive any notifications. We don't recommend ignoring repositories as you won't be notified if you're @mentioned. {% if currentVersion == "free-pro-team@latest" %}If you experiencing abuse and want to ignore a repository, please contact {% data variables.contact.contact_support %} so we can help. {% data reusables.policies.abuse %}{% endif %}
|
||||
|
||||
{% endnote %}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ redirect_from:
|
|||
versions:
|
||||
free-pro-team: '*'
|
||||
enterprise-server: '*'
|
||||
github-ae: '*'
|
||||
---
|
||||
|
||||
<div class="jumbotron libraries-jumbotron">
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ http(s)://<em>hostname</em>/
|
|||
{% if currentVersion == "github-ae@latest" or enterpriseServerVersions contains currentVersion %}
|
||||
### Authentication
|
||||
|
||||
Your {% data variables.product.product_name %} installation's API endpoints accept [the same authentication methods](/rest/overview/resources-in-the-rest-api#authentication) as the GitHub.com API. You can authenticate yourself with **[OAuth tokens](/apps/building-integrations/setting-up-and-registering-oauth-apps/)** (which can be created using the [Authorizations API](/rest/reference/oauth-authorizations#create-a-new-authorization)) or **[basic authentication](/rest/overview/resources-in-the-rest-api#basic-authentication)**. {% if enterpriseServerVersions contains currentVersion %}
|
||||
Your {% data variables.product.product_name %} installation's API endpoints accept [the same authentication methods](/rest/overview/resources-in-the-rest-api#authentication) as the GitHub.com API. You can authenticate yourself with **[OAuth tokens](/apps/building-integrations/setting-up-and-registering-oauth-apps/)** {% if enterpriseServerVersions contains currentVersion %}(which can be created using the [Authorizations API](/rest/reference/oauth-authorizations#create-a-new-authorization)) {% endif %}or **[basic authentication](/rest/overview/resources-in-the-rest-api#basic-authentication)**. {% if enterpriseServerVersions contains currentVersion %}
|
||||
OAuth tokens must have the `site_admin` [OAuth scope](/developers/apps/scopes-for-oauth-apps#available-scopes) when used with Enterprise-specific endpoints.{% endif %}
|
||||
|
||||
Enterprise administration API endpoints are only accessible to authenticated {% data variables.product.product_name %} site administrators{% if enterpriseServerVersions contains currentVersion %}, except for the [Management Console](#management-console) API, which requires the [Management Console password](/enterprise/admin/articles/accessing-the-management-console/){% endif %}.
|
||||
|
|
|
|||
|
|
@ -1,4 +1 @@
|
|||
{% if currentVersion == "free-pro-team@latest" or currentVersion ver_gt "enterprise-server@2.21" or currentVersion == "github-ae@latest" %}
|
||||
Suspending a {% data variables.product.prodname_github_app %} installation is currently in beta and subject to change. Before you can suspend a {% data variables.product.prodname_github_app %}, the app owner must enable suspending installations for the app by opting-in to the beta. To opt-in to the suspending installations beta feature, see "[Activating beta features for apps](/developers/apps/activating-beta-features-for-apps)."
|
||||
|
||||
{% endif %}
|
||||
|
|
|
|||
|
|
@ -1,137 +0,0 @@
|
|||
const cheerio = require('cheerio')
|
||||
const findPageInVersion = require('./find-page-in-version')
|
||||
const renderContent = require('./render-content')
|
||||
const rewriteLocalLinks = require('./rewrite-local-links')
|
||||
const nonEnterpriseDefaultVersion = require('./non-enterprise-default-version')
|
||||
const { getPathWithoutLanguage } = require('./path-utils')
|
||||
const { getEnterpriseVersionNumber, adminProduct } = require('./patterns')
|
||||
const { deprecated, latest } = require('./enterprise-server-releases')
|
||||
|
||||
// internal links will have a language code by the time we're testing them
|
||||
// we also want to capture same-page anchors (#foo)
|
||||
const languageCode = 'en'
|
||||
const internalHrefs = ['/en', '#']
|
||||
|
||||
const renderedPageCache = {}
|
||||
const checkedAnchorCache = {}
|
||||
|
||||
module.exports = async function checkLinks ($, page, context, version, checkedLinkCache = {}) {
|
||||
// run rewriteLocalLinks to version links and add language codes
|
||||
rewriteLocalLinks($, version, languageCode)
|
||||
|
||||
const brokenLinks = {
|
||||
anchors: [],
|
||||
links: []
|
||||
}
|
||||
|
||||
// internal link check
|
||||
for (const href of internalHrefs) {
|
||||
const internalLinks = $(`a[href^="${href}"]`).get()
|
||||
|
||||
for (const internalLink of internalLinks) {
|
||||
const href = $(internalLink).attr('href')
|
||||
|
||||
// enable caching so we don't check links more than once
|
||||
// anchor links are cached locally (within this run) since they are specific to the page
|
||||
if (checkedLinkCache[href] || checkedAnchorCache[href]) continue
|
||||
|
||||
const [link, anchor] = href.split('#')
|
||||
|
||||
// if anchor only (e.g., #foo), look for heading on same page
|
||||
if (anchor && !link) {
|
||||
// ignore anchors that are autogenerated from headings
|
||||
if (anchor === $(internalLink).parent().attr('id')) continue
|
||||
|
||||
const matchingHeadings = getMatchingHeadings($, anchor)
|
||||
|
||||
if (matchingHeadings.length === 0) {
|
||||
brokenLinks.anchors.push({ 'broken same-page anchor': `#${anchor}`, reason: 'heading not found on page' })
|
||||
}
|
||||
checkedAnchorCache[href] = true
|
||||
continue
|
||||
}
|
||||
checkedLinkCache[href] = true
|
||||
|
||||
// skip rare hardcoded links to old GHE versions
|
||||
// these paths will always be in the old versioned form
|
||||
// example: /enterprise/11.10.340/admin/articles/upgrading-to-the-latest-release
|
||||
const gheVersionInLink = link.match(getEnterpriseVersionNumber)
|
||||
if (gheVersionInLink && deprecated.includes(gheVersionInLink[1])) continue
|
||||
|
||||
// look for linked page
|
||||
const isDotcomOnly = $(internalLink).attr('class')
|
||||
|
||||
// special case for GHES Admin links on dotcom, which are not broken; they go to the latest GHES version
|
||||
let versionToCheck = version
|
||||
if (version === nonEnterpriseDefaultVersion && adminProduct.test(link)) {
|
||||
versionToCheck = `enterprise-server@${latest}`
|
||||
}
|
||||
|
||||
const linkedPage = findPageInVersion(link, context.pages, context.redirects, languageCode, versionToCheck, isDotcomOnly)
|
||||
|
||||
if (!linkedPage) {
|
||||
brokenLinks.links.push({ 'broken link': link, reason: 'linked page not found' })
|
||||
continue
|
||||
}
|
||||
|
||||
if (linkedPage.relativePath.includes('rest/reference/') && linkedPage.relativePath !== 'rest/reference/index.md') {
|
||||
const linkedPageRelevantPermalink = linkedPage.permalinks.find(permalink => permalink.pageVersion === version)
|
||||
if (!linkedPageRelevantPermalink) continue
|
||||
|
||||
const docsPath = linkedPageRelevantPermalink.href
|
||||
.split('rest/reference/')[1]
|
||||
.split('#')[0] // do not include #fragments
|
||||
|
||||
// find all operations that with an operationID that matches the requested docs path
|
||||
context.currentRestOperations = context.operationsForCurrentProduct
|
||||
.filter(operation => operation.operationId.startsWith(docsPath))
|
||||
}
|
||||
|
||||
// collect elements of the page that may contain links
|
||||
const linkedPageContent = linkedPage.relativePath.includes('graphql/reference/objects')
|
||||
? linkedPage.markdown + context.graphql.prerenderedObjectsForCurrentVersion.html
|
||||
: linkedPage.markdown
|
||||
|
||||
// create a unique string for caching purposes
|
||||
const pathToCache = version + linkedPage.relativePath
|
||||
|
||||
const anchorToCheck = anchor
|
||||
|
||||
// if link with anchor (e.g., /some/path#foo), look for heading on linked page
|
||||
if (anchorToCheck) {
|
||||
// either render page or fetch it from cache if we've already rendered it
|
||||
let linkedPageObject
|
||||
if (!renderedPageCache[pathToCache]) {
|
||||
const linkedPageHtml = await renderContent(linkedPageContent, context)
|
||||
linkedPageObject = cheerio.load(linkedPageHtml, { xmlMode: true })
|
||||
renderedPageCache[pathToCache] = linkedPageObject
|
||||
} else {
|
||||
linkedPageObject = renderedPageCache[pathToCache]
|
||||
}
|
||||
|
||||
const matchingHeadings = getMatchingHeadings(linkedPageObject, anchorToCheck)
|
||||
|
||||
if (matchingHeadings.length === 0) {
|
||||
if (anchor) {
|
||||
brokenLinks.anchors.push({ 'broken anchor': `#${anchor}`, 'full link': `${getPathWithoutLanguage(link)}#${anchor}`, reason: 'heading not found on linked page', 'linked page': linkedPage.fullPath })
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { brokenLinks, checkedLinkCache }
|
||||
}
|
||||
|
||||
// article titles are h1s; headings can be any subsequent level
|
||||
function getMatchingHeadings ($, anchor) {
|
||||
return $(`
|
||||
h2[id="${anchor}"],
|
||||
h3[id="${anchor}"],
|
||||
h4[id="${anchor}"],
|
||||
h5[id="${anchor}"],
|
||||
h6[id="${anchor}"],
|
||||
a[name="${anchor}"]
|
||||
`).get()
|
||||
}
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
const rewriteAssetPathsToS3 = require('./rewrite-asset-paths-to-s3')
|
||||
const { promisify } = require('util')
|
||||
|
||||
module.exports = async function checkImages ($, version, relativePath, checkedImageCache = {}) {
|
||||
rewriteAssetPathsToS3($, version, relativePath)
|
||||
|
||||
const brokenImages = []
|
||||
|
||||
// this does not check S3 images because those live outside of the repo
|
||||
const images = $('img[src^="/assets"]').get()
|
||||
|
||||
for (const image of images) {
|
||||
const src = $(image).attr('src')
|
||||
|
||||
if (checkedImageCache[src]) continue
|
||||
|
||||
try {
|
||||
await promisify(fs.access)(path.join(__dirname, '..', src))
|
||||
} catch (e) {
|
||||
brokenImages.push({ 'broken image reference': src })
|
||||
}
|
||||
}
|
||||
|
||||
return { brokenImages, checkedImageCache }
|
||||
}
|
||||
|
|
@ -1,122 +0,0 @@
|
|||
const cheerio = require('cheerio')
|
||||
const findPageInVersion = require('./find-page-in-version')
|
||||
const renderContent = require('./render-content')
|
||||
const rewriteLocalLinks = require('./rewrite-local-links')
|
||||
const nonEnterpriseDefaultVersion = require('./non-enterprise-default-version')
|
||||
const { getPathWithoutLanguage } = require('./path-utils')
|
||||
const { getEnterpriseVersionNumber, adminProduct } = require('./patterns')
|
||||
const { deprecated, latest } = require('./enterprise-server-releases')
|
||||
|
||||
// internal links will have a language code by the time we're testing them
|
||||
// we also want to capture same-page anchors (#foo)
|
||||
const languageCode = 'en'
|
||||
const internalHrefs = ['/en', '#']
|
||||
|
||||
const renderedPageCache = {}
|
||||
const checkedAnchorCache = {}
|
||||
|
||||
module.exports = async function checkLinks ($, page, context, version, checkedLinkCache = {}) {
|
||||
// run rewriteLocalLinks to version links and add language codes
|
||||
rewriteLocalLinks($, version, languageCode)
|
||||
|
||||
const brokenLinks = {
|
||||
anchors: [],
|
||||
links: []
|
||||
}
|
||||
|
||||
// internal link check
|
||||
for (const href of internalHrefs) {
|
||||
const internalLinks = $(`a[href^="${href}"]`).get()
|
||||
|
||||
for (const internalLink of internalLinks) {
|
||||
const href = $(internalLink).attr('href')
|
||||
|
||||
// enable caching so we don't check links more than once
|
||||
// anchor links are cached locally (within this run) since they are specific to the page
|
||||
if (checkedLinkCache[href] || checkedAnchorCache[href]) continue
|
||||
|
||||
const [link, anchor] = href.split('#')
|
||||
|
||||
// if anchor only (e.g., #foo), look for heading on same page
|
||||
if (anchor && !link) {
|
||||
// ignore anchors that are autogenerated from headings
|
||||
if (anchor === $(internalLink).parent().attr('id')) continue
|
||||
|
||||
const matchingHeadings = getMatchingHeadings($, anchor)
|
||||
|
||||
if (matchingHeadings.length === 0) {
|
||||
brokenLinks.anchors.push({ 'broken same-page anchor': `#${anchor}`, reason: 'heading not found on page' })
|
||||
}
|
||||
checkedAnchorCache[href] = true
|
||||
continue
|
||||
}
|
||||
checkedLinkCache[href] = true
|
||||
|
||||
// skip rare hardcoded links to old GHE versions
|
||||
// these paths will always be in the old versioned form
|
||||
// example: /enterprise/11.10.340/admin/articles/upgrading-to-the-latest-release
|
||||
const gheVersionInLink = link.match(getEnterpriseVersionNumber)
|
||||
if (gheVersionInLink && deprecated.includes(gheVersionInLink[1])) continue
|
||||
|
||||
// look for linked page
|
||||
const isDotcomOnly = $(internalLink).attr('class')
|
||||
|
||||
// special case for GHES Admin links on dotcom, which are not broken; they go to the latest GHES version
|
||||
let versionToCheck = version
|
||||
if (version === nonEnterpriseDefaultVersion && adminProduct.test(link)) {
|
||||
versionToCheck = `enterprise-server@${latest}`
|
||||
}
|
||||
|
||||
const linkedPage = findPageInVersion(link, context.pages, context.redirects, languageCode, versionToCheck, isDotcomOnly)
|
||||
|
||||
if (!linkedPage) {
|
||||
brokenLinks.links.push({ 'broken link': link, reason: 'linked page not found' })
|
||||
continue
|
||||
}
|
||||
|
||||
// don't check anchors on developers content
|
||||
if (linkedPage.relativePath.match(/^(rest|graphql|developers)/)) continue
|
||||
|
||||
// create a unique string for caching purposes
|
||||
const pathToCache = version + linkedPage.relativePath
|
||||
|
||||
const anchorToCheck = anchor
|
||||
|
||||
// if link with anchor (e.g., /some/path#foo), look for heading on linked page
|
||||
if (anchorToCheck) {
|
||||
// either render page or fetch it from cache if we've already rendered it
|
||||
let linkedPageObject
|
||||
if (!renderedPageCache[pathToCache]) {
|
||||
const linkedPageHtml = await renderContent(linkedPage.markdown, context)
|
||||
linkedPageObject = cheerio.load(linkedPageHtml, { xmlMode: true })
|
||||
renderedPageCache[pathToCache] = linkedPageObject
|
||||
} else {
|
||||
linkedPageObject = renderedPageCache[pathToCache]
|
||||
}
|
||||
|
||||
const matchingHeadings = getMatchingHeadings(linkedPageObject, anchorToCheck)
|
||||
|
||||
if (matchingHeadings.length === 0) {
|
||||
if (anchor) {
|
||||
brokenLinks.anchors.push({ 'broken anchor': `#${anchor}`, 'full link': `${getPathWithoutLanguage(link)}#${anchor}`, reason: 'heading not found on linked page', 'linked page': linkedPage.fullPath })
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { brokenLinks, checkedLinkCache }
|
||||
}
|
||||
|
||||
// article titles are h1s; headings can be any subsequent level
|
||||
function getMatchingHeadings ($, anchor) {
|
||||
return $(`
|
||||
h2[id="${anchor}"],
|
||||
h3[id="${anchor}"],
|
||||
h4[id="${anchor}"],
|
||||
h5[id="${anchor}"],
|
||||
h6[id="${anchor}"],
|
||||
a[name="${anchor}"]
|
||||
`)
|
||||
}
|
||||
|
|
@ -1,42 +1,52 @@
|
|||
const slash = require('slash')
|
||||
const path = require('path')
|
||||
const patterns = require('./patterns')
|
||||
const { deprecated } = require('./enterprise-server-releases')
|
||||
const { deprecated, latest } = require('./enterprise-server-releases')
|
||||
const allProducts = require('./all-products')
|
||||
const allVersions = require('./all-versions')
|
||||
const supportedVersions = new Set(Object.keys(allVersions))
|
||||
const { getNewVersionedPath } = require('./old-versions-utils')
|
||||
|
||||
const supportedVersions = new Set(Object.keys(allVersions))
|
||||
|
||||
// construct appropriate versioned path for any given HREF
|
||||
// This function constructs an appropriate versioned path for any given HREF.
|
||||
// NOTE: this gets called by findPage and various other functions, and
|
||||
// has to return a proper versioned link given a wide variety of incoming
|
||||
// modern or legacy-formatted links, so it is somewhat overloaded. At some point
|
||||
// this could probably be broken up into separate functions to handle different incoming
|
||||
// paths. But it is currently optimized to handle lots of edge cases.
|
||||
function getVersionedPathWithoutLanguage (href, version) {
|
||||
// start clean without language code or trailing slash
|
||||
// Start clean without language code or trailing slash
|
||||
href = getPathWithoutLanguage(href.replace(patterns.trailingSlash, '$1'))
|
||||
|
||||
// if this is an old versioned path that includes a deprecated version, do not change!
|
||||
// If this is an old versioned path that includes a deprecated version, do not change!
|
||||
// example: /enterprise/11.10.340/admin/articles/upgrading-to-the-latest-release
|
||||
const oldEnterpriseVersionNumber = href.match(patterns.getEnterpriseVersionNumber)
|
||||
if (oldEnterpriseVersionNumber && deprecated.includes(oldEnterpriseVersionNumber[1])) {
|
||||
return href
|
||||
}
|
||||
|
||||
// try to derive the current version from the path
|
||||
// Try to derive the current version from the path
|
||||
// example: enterprise-server@2.22 or free-pro-team@latest
|
||||
let versionFromPath = getVersionStringFromPath(href)
|
||||
|
||||
// if the version found is not a currently supported version...
|
||||
// If a supported version was found, add it to the path so we can go through the rest of the checks
|
||||
if (supportedVersions.has(versionFromPath)) {
|
||||
href = href.replace(href.split('/')[1], versionFromPath)
|
||||
}
|
||||
|
||||
// If a currently supported version was NOT found...
|
||||
let productObjectFromPath
|
||||
if (!supportedVersions.has(versionFromPath)) {
|
||||
// first check if the first segment is instead a current product;
|
||||
// First check if the segment is instead a current product;
|
||||
// example: /admin/foo or /desktop/foo
|
||||
productObjectFromPath = allProducts[versionFromPath]
|
||||
|
||||
// if so, add the first supported version for that product to the href
|
||||
// If so, add the first supported version for that product to the href
|
||||
// (this is just to get a path with all the expected segments; the version will be updated later if needed)
|
||||
if (productObjectFromPath) {
|
||||
href = path.join('/', productObjectFromPath.versions[0], href)
|
||||
versionFromPath = productObjectFromPath.versions[0]
|
||||
} else {
|
||||
// otherwise, this may be an old path that should be converted to new path;
|
||||
// Otherwise, this may be an old path that should be converted to new path;
|
||||
// OLD: /enterprise/2.22/admin/installation OR /enterprise/admin/installation
|
||||
// NEW: /enterprise-server@2.22/admin/installation
|
||||
href = getNewVersionedPath(href)
|
||||
|
|
@ -44,58 +54,83 @@ function getVersionedPathWithoutLanguage (href, version) {
|
|||
}
|
||||
}
|
||||
|
||||
// if not previously found, derive the product object from the path (e.g., github or admin)
|
||||
// If not previously found, derive the product object from the path (e.g., github or admin)
|
||||
if (!productObjectFromPath) {
|
||||
productObjectFromPath = getProductObjectFromPath(href)
|
||||
}
|
||||
|
||||
// if the product's versions don't include the specified version, nothing to change!
|
||||
// If the product's versions don't include the specified version, nothing to change!
|
||||
if (productObjectFromPath && !productObjectFromPath.versions.includes(version)) {
|
||||
return slash(href)
|
||||
}
|
||||
|
||||
// update the version
|
||||
// Update the version and return the path
|
||||
return slash(href.replace(versionFromPath, version))
|
||||
}
|
||||
|
||||
// add language code
|
||||
// Add language code to a versioned path
|
||||
function getVersionedPathWithLanguage (href, version, languageCode) {
|
||||
return getPathWithLanguage(getVersionedPathWithoutLanguage(href, version), languageCode)
|
||||
}
|
||||
|
||||
// add the language to the given HREF
|
||||
// /en/articles/foo -> /articles/foo
|
||||
// Add the language to the given HREF
|
||||
// /articles/foo -> /en/articles/foo
|
||||
function getPathWithLanguage (href, languageCode) {
|
||||
return slash(path.posix.join('/', languageCode, getPathWithoutLanguage(href)))
|
||||
.replace(patterns.trailingSlash, '$1')
|
||||
}
|
||||
|
||||
// remove the language from the given HREF
|
||||
// /articles/foo -> /en/articles/foo
|
||||
// Remove the language from the given HREF
|
||||
// /en/articles/foo -> /articles/foo
|
||||
function getPathWithoutLanguage (href) {
|
||||
return slash(href.replace(patterns.hasLanguageCode, '/'))
|
||||
}
|
||||
|
||||
// Remove the version segment from the path
|
||||
function getPathWithoutVersion (href) {
|
||||
return href.replace(`/${getVersionStringFromPath(href)}`, '')
|
||||
}
|
||||
|
||||
// Return the version segment in a path
|
||||
function getVersionStringFromPath (href) {
|
||||
href = getPathWithoutLanguage(href)
|
||||
const versionString = href.split('/')[1]
|
||||
|
||||
return versionString || 'homepage'
|
||||
// Return immediately if this is a link to the homepage
|
||||
if (href === '/') {
|
||||
return 'homepage'
|
||||
}
|
||||
|
||||
// Check if the first segment is a supported version
|
||||
const versionFromPath = href.split('/')[1]
|
||||
|
||||
if (supportedVersions.has(versionFromPath)) {
|
||||
return versionFromPath
|
||||
}
|
||||
|
||||
// If the version segment is the latest enterprise-server release, return the latest release
|
||||
if (versionFromPath === 'enterprise-server@latest') {
|
||||
return `enterprise-server@${latest}`
|
||||
}
|
||||
|
||||
// If it's just a plan with no @release (e.g., `enterprise-server`), return the latest release
|
||||
const planObject = Object.values(allVersions).find(v => v.plan === versionFromPath)
|
||||
if (planObject) {
|
||||
return allVersions[planObject.latestVersion].version
|
||||
}
|
||||
|
||||
// Otherwise, return the first segment as-is, which may not be a real supported version,
|
||||
// but additional checks are done on this segment in getVersionedPathWithoutLanguage
|
||||
return versionFromPath
|
||||
}
|
||||
|
||||
// Return the corresponding object for the version segment in a path
|
||||
function getVersionObjectFromPath (href) {
|
||||
const versionId = getVersionStringFromPath(href)
|
||||
const version = allVersions[versionId]
|
||||
const versionFromPath = getVersionStringFromPath(href)
|
||||
|
||||
if (!version) throw new Error(`No version found for ${href}`)
|
||||
|
||||
return version
|
||||
return allVersions[versionFromPath]
|
||||
}
|
||||
|
||||
// Return the product segment from the path
|
||||
function getProductStringFromPath (href) {
|
||||
href = getPathWithoutLanguage(href)
|
||||
const productString = href.split('/')[2]
|
||||
|
|
@ -103,10 +138,11 @@ function getProductStringFromPath (href) {
|
|||
return productString || 'homepage'
|
||||
}
|
||||
|
||||
// Return the corresponding object for the product segment in a path
|
||||
function getProductObjectFromPath (href) {
|
||||
const productId = getProductStringFromPath(href)
|
||||
// Return undefined if product id derived from path can't be found in allProducts
|
||||
return allProducts[productId]
|
||||
const productFromPath = getProductStringFromPath(href)
|
||||
|
||||
return allProducts[productFromPath]
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,270 @@
|
|||
const cheerio = require('cheerio')
|
||||
const { union, uniq } = require('lodash')
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
|
||||
const { getVersionStringFromPath } = require('../../lib/path-utils')
|
||||
const patterns = require('../../lib/patterns')
|
||||
const { deprecated } = require('../../lib/enterprise-server-releases')
|
||||
const findPageInVersion = require('../../lib/find-page-in-version')
|
||||
const rest = require('../../middleware/contextualizers/rest')
|
||||
const graphql = require('../../middleware/contextualizers/graphql')
|
||||
const contextualize = require('../../middleware/context')
|
||||
const releaseNotes = require('../../middleware/contextualizers/enterprise-release-notes')
|
||||
const versionSatisfiesRange = require('../../lib/version-satisfies-range')
|
||||
|
||||
class LinksChecker {
|
||||
constructor (opts = { languageCode: 'en', internalHrefPrefixes: ['/', '#'] }) {
|
||||
Object.assign(this, { ...opts })
|
||||
|
||||
// Some caching mechanism so we do not load pages unnecessarily,
|
||||
// nor check links that have been checked
|
||||
this.pageCache = new Map()
|
||||
this.checkedLinksCache = new Set()
|
||||
|
||||
// stores images to check all at once in a Map:
|
||||
// imageSrc => {
|
||||
// "usedBy": [version:path, ...]
|
||||
// }
|
||||
this.imagesToCheck = new Map()
|
||||
|
||||
// Stores broken images in a Map, formatted the same way as imagesToCheck
|
||||
this.brokenImages = new Map()
|
||||
|
||||
// Stores broken links in a Map in the format of:
|
||||
// link => {
|
||||
// linkedFrom: [ version:filePath, ... ]
|
||||
// }, ...
|
||||
this.brokenLinks = new Map()
|
||||
|
||||
// stores anchor links to check all at once in a Map:
|
||||
// version:filePath => {
|
||||
// '#anchor-link' : {
|
||||
// linkedFrom: ['url1', 'url2']
|
||||
// },
|
||||
// '#anchor-link2': {...}
|
||||
// }
|
||||
this.anchorLinksToCheck = new Map()
|
||||
|
||||
// Stores broken anchors in a Map, formatted the same way as anchorLinksToCheck
|
||||
this.brokenAnchors = new Map()
|
||||
}
|
||||
|
||||
async setRenderedPageObj (pathCacheKey, context, reRender = false) {
|
||||
if (this.pageCache.has(pathCacheKey) && !reRender) return
|
||||
let pageHTML = await context.page.render(context)
|
||||
|
||||
// handle special pre-rendered snowflake
|
||||
if (context.page.relativePath.endsWith('graphql/reference/objects.md')) {
|
||||
pageHTML += context.graphql.prerenderedObjectsForCurrentVersion.html
|
||||
}
|
||||
|
||||
const pageObj = cheerio.load(pageHTML, { xmlMode: true })
|
||||
this.pageCache.set(pathCacheKey, pageObj)
|
||||
}
|
||||
|
||||
async getRenderedPageObj (pathCacheKey, context) {
|
||||
if (!this.pageCache.has(pathCacheKey)) {
|
||||
if (context) {
|
||||
await this.setRenderedPageObj(pathCacheKey, context)
|
||||
} else {
|
||||
console.error('cannot find pre-rendered page, and does not have enough context to render one.')
|
||||
}
|
||||
}
|
||||
return this.pageCache.get(pathCacheKey)
|
||||
}
|
||||
|
||||
addAnchorForLater (pagePath, anchor, linkedFrom) {
|
||||
const anchorsInPath = this.anchorLinksToCheck.get(pagePath) || {}
|
||||
const anchorLink = anchorsInPath[anchor] || { linkedFrom: [] }
|
||||
anchorLink.linkedFrom = union(anchorLink.linkedFrom, [linkedFrom])
|
||||
anchorsInPath[anchor] = anchorLink
|
||||
this.anchorLinksToCheck.set(pagePath, anchorsInPath)
|
||||
}
|
||||
|
||||
addImagesForLater (images, pagePath) {
|
||||
uniq(images).forEach(imageSrc => {
|
||||
const imageUsage = this.imagesToCheck.get(imageSrc) || { usedBy: [] }
|
||||
imageUsage.usedBy = union(imageUsage.usedBy, [pagePath])
|
||||
this.imagesToCheck.set(imageSrc, imageUsage)
|
||||
})
|
||||
}
|
||||
|
||||
async checkPage (context, checkExternalAnchors) {
|
||||
const path = context.relativePath
|
||||
const version = context.currentVersion
|
||||
|
||||
const pathCacheKey = `${version}:${path}`
|
||||
const $ = await this.getRenderedPageObj(pathCacheKey, context)
|
||||
|
||||
const imageSrcs = $('img[src^="/assets"]').map((i, el) => $(el).attr('src')).toArray()
|
||||
|
||||
this.addImagesForLater(imageSrcs, pathCacheKey)
|
||||
|
||||
for (const href of this.internalHrefPrefixes) {
|
||||
const internalLinks = $(`a[href^="${href}"]`).get()
|
||||
|
||||
for (const internalLink of internalLinks) {
|
||||
const href = $(internalLink).attr('href')
|
||||
|
||||
let [link, anchor] = href.split('#')
|
||||
// remove trailing slash
|
||||
link = link.replace(patterns.trailingSlash, '$1')
|
||||
|
||||
// if it's an external link and has been checked before, skip
|
||||
if (link && this.checkedLinksCache.has(link)) {
|
||||
// if it's been determined this link is broken, add to the linkedFrom field
|
||||
if (this.brokenLinks.has(link)) {
|
||||
const brokenLink = this.brokenLinks.get(link)
|
||||
brokenLink.linkedFrom = union(brokenLink.linkedFrom, [pathCacheKey])
|
||||
this.brokenLinks.set(link, brokenLink)
|
||||
}
|
||||
if (!anchor) continue
|
||||
}
|
||||
|
||||
// if it's an internal anchor (e.g., #foo), save for later
|
||||
if (anchor && !link) {
|
||||
// ignore anchors that are autogenerated from headings
|
||||
if (anchor === $(internalLink).parent().attr('id')) continue
|
||||
this.addAnchorForLater(pathCacheKey, anchor, 'same page')
|
||||
continue
|
||||
}
|
||||
|
||||
// ------ BEGIN ONEOFF EXCLUSIONS -------///
|
||||
// skip GraphQL public schema paths (these are checked by separate tests)
|
||||
if (link.startsWith('/public/') && link.endsWith('.graphql')) continue
|
||||
|
||||
// skip links that start with /assets/images, as these are not in the pages collection
|
||||
// and /assets/images paths should be checked during the image check
|
||||
if (link.startsWith('/assets/images')) continue
|
||||
|
||||
// skip rare hardcoded links to old GHE versions
|
||||
// these paths will always be in the old versioned format
|
||||
// example: /enterprise/11.10.340/admin/articles/upgrading-to-the-latest-release
|
||||
const gheVersionInLink = link.match(patterns.getEnterpriseVersionNumber)
|
||||
if (gheVersionInLink && deprecated.includes(gheVersionInLink[1])) continue
|
||||
// ------ END ONEOFF EXCLUSIONS -------///
|
||||
|
||||
// the link at this point should include a version via lib/rewrite-local-links
|
||||
const versionFromHref = getVersionStringFromPath(link)
|
||||
|
||||
// look for linked page
|
||||
const linkedPage = findPageInVersion(link, context.pages, context.redirects, this.languageCode, versionFromHref)
|
||||
this.checkedLinksCache.add(link)
|
||||
|
||||
if (!linkedPage) {
|
||||
this.brokenLinks.set(link, { linkedFrom: [pathCacheKey] })
|
||||
continue
|
||||
}
|
||||
|
||||
// if we're not checking external anchors, we're done
|
||||
if (!checkExternalAnchors) {
|
||||
continue
|
||||
}
|
||||
|
||||
// find the permalink for the current version
|
||||
const linkedPagePermalink = linkedPage.permalinks.find(permalink => permalink.pageVersion === version)
|
||||
|
||||
if (linkedPagePermalink) {
|
||||
const linkedPageContext = await buildPathContext(context, linkedPage, linkedPagePermalink)
|
||||
|
||||
if (anchor) {
|
||||
await this.setRenderedPageObj(`${version}:${linkedPage.relativePath}`, linkedPageContext)
|
||||
this.addAnchorForLater(`${version}:${linkedPage.relativePath}`, anchor, pathCacheKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async checkAnchors () {
|
||||
for await (const [pathCacheKey, anchors] of this.anchorLinksToCheck) {
|
||||
const $ = await this.getRenderedPageObj(pathCacheKey)
|
||||
for (const anchorText in anchors) {
|
||||
const matchingHeadings = $(`[id="${anchorText}"], [name="${anchorText}"]`)
|
||||
if (matchingHeadings.length === 0) {
|
||||
const brokenAnchorPath = this.brokenAnchors.get(pathCacheKey) || {}
|
||||
brokenAnchorPath[anchorText] = anchors[anchorText]
|
||||
this.brokenAnchors.set(pathCacheKey, brokenAnchorPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
getBrokenLinks () {
|
||||
return this.brokenLinks
|
||||
}
|
||||
|
||||
async getBrokenAnchors () {
|
||||
await this.checkAnchors()
|
||||
return this.brokenAnchors
|
||||
}
|
||||
|
||||
async getBrokenImages () {
|
||||
for await (const [imageSrc, imageUsage] of this.imagesToCheck) {
|
||||
try {
|
||||
await fs.promises.access(path.join(process.cwd(), imageSrc))
|
||||
} catch (e) {
|
||||
this.brokenImages.set(imageSrc, imageUsage)
|
||||
}
|
||||
}
|
||||
return this.brokenImages
|
||||
}
|
||||
}
|
||||
|
||||
// this function is async because the middleware functions are likely async
|
||||
async function applyMiddleware (middleware, req) {
|
||||
return middleware(req, null, () => {})
|
||||
}
|
||||
|
||||
async function buildInitialContext () {
|
||||
const req = {
|
||||
path: '/en',
|
||||
language: 'en',
|
||||
query: {}
|
||||
}
|
||||
await applyMiddleware(contextualize, req)
|
||||
return req.context
|
||||
}
|
||||
|
||||
async function buildPathContext (initialContext, page, permalink) {
|
||||
// Create a new object with path-specific properties.
|
||||
// Note this is cherry-picking properties currently only needed by the middlware below;
|
||||
// See middleware/context.js for the rest of the properties we are NOT refreshing per page.
|
||||
// If we find this causes problems for link checking, we can call `contextualize` on
|
||||
// every page. For now, this cherry-picking approach is intended to improve performance so
|
||||
// we don't have to build the expensive `pages`, `redirects`, etc. data on every page we check.
|
||||
const pathContext = {
|
||||
page,
|
||||
currentVersion: permalink.pageVersion,
|
||||
relativePath: permalink.relativePath
|
||||
}
|
||||
|
||||
// Combine it with the initial context object that has pages, redirects, etc.
|
||||
const combinedContext = Object.assign({}, initialContext, pathContext)
|
||||
|
||||
// Create a new req object using the combined context
|
||||
const req = {
|
||||
path: permalink.href,
|
||||
context: combinedContext,
|
||||
language: 'en',
|
||||
query: {}
|
||||
}
|
||||
|
||||
// Pass the req to the contextualizing middlewares
|
||||
await applyMiddleware(rest, req)
|
||||
await applyMiddleware(graphql, req)
|
||||
// Release notes are available on docs site starting with GHES 3.0
|
||||
if (versionSatisfiesRange(permalink.pageVersion, '>=3.0')) {
|
||||
await applyMiddleware(releaseNotes, req)
|
||||
}
|
||||
|
||||
// Return the resulting context object with REST, GraphQL, and release notes data now attached
|
||||
return req.context
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
LinksChecker,
|
||||
buildPathContext,
|
||||
buildInitialContext
|
||||
}
|
||||
|
|
@ -1,158 +0,0 @@
|
|||
const flat = require('flat')
|
||||
const { last } = require('lodash')
|
||||
const cheerio = require('cheerio')
|
||||
const { loadPages, loadPageMap } = require('../../lib/pages')
|
||||
const loadSiteData = require('../../lib/site-data')
|
||||
const getApplicableVersions = require('../../lib/get-applicable-versions')
|
||||
const loadRedirects = require('../../lib/redirects/precompile')
|
||||
const { getVersionedPathWithLanguage } = require('../../lib/path-utils')
|
||||
const renderContent = require('../../lib/render-content')
|
||||
const checkImages = require('../../lib/check-images')
|
||||
const checkLinks = require('../../lib/check-developer-links')
|
||||
const allVersions = require('../../lib/all-versions')
|
||||
const enterpriseServerVersions = Object.keys(require('../../lib/all-versions'))
|
||||
.filter(version => version.startsWith('enterprise-server@'))
|
||||
|
||||
// schema-derived data to add to context object
|
||||
const rest = require('../../lib/rest')
|
||||
const previews = require('../../lib/graphql/static/previews')
|
||||
const upcomingChanges = require('../../lib/graphql/static/upcoming-changes')
|
||||
const changelog = require('../../lib/graphql/static/changelog')
|
||||
const prerenderedObjects = require('../../lib/graphql/static/prerendered-objects')
|
||||
|
||||
// english only
|
||||
const languageCode = 'en'
|
||||
|
||||
const context = {
|
||||
currentLanguage: languageCode,
|
||||
rest
|
||||
}
|
||||
|
||||
// developer content only
|
||||
const developerContentRegex = /^(rest|graphql|developers)/
|
||||
|
||||
describe('page rendering', () => {
|
||||
jest.setTimeout(1000 * 1000)
|
||||
|
||||
const brokenImages = {}
|
||||
const brokenAnchors = {}
|
||||
const brokenLinks = {}
|
||||
|
||||
beforeAll(async (done) => {
|
||||
const pageList = await loadPages()
|
||||
const pageMap = await loadPageMap(pageList)
|
||||
const siteData = await loadSiteData()
|
||||
const redirects = await loadRedirects(pageList, pageMap)
|
||||
|
||||
context.pages = pageMap
|
||||
context.site = siteData[languageCode].site
|
||||
context.redirects = redirects
|
||||
|
||||
const developerPages = pageList
|
||||
.filter(page => page.relativePath.match(developerContentRegex) && page.languageCode === languageCode)
|
||||
|
||||
let checkedLinks = {}
|
||||
let checkedImages = {}
|
||||
|
||||
for (const page of developerPages) {
|
||||
const brokenImagesPerPage = {}
|
||||
const brokenAnchorsPerPage = {}
|
||||
const brokenLinksPerPage = {}
|
||||
|
||||
// get an array of the pages product versions
|
||||
const pageVersions = getApplicableVersions(page.versions, page.relativePath)
|
||||
|
||||
for (const pageVersion of pageVersions) {
|
||||
// attach page-specific properties to context
|
||||
page.version = pageVersion
|
||||
context.page = page
|
||||
context.currentVersion = pageVersion
|
||||
context.enterpriseServerVersions = enterpriseServerVersions
|
||||
|
||||
const relevantPermalink = page.permalinks.find(permalink => permalink.pageVersion === pageVersion)
|
||||
|
||||
const graphqlVersion = allVersions[pageVersion].miscVersionName
|
||||
|
||||
// borrowed from middleware/contextualizers/graphql.js
|
||||
context.graphql = {
|
||||
schemaForCurrentVersion: require(`../../lib/graphql/static/schema-${graphqlVersion}`),
|
||||
previewsForCurrentVersion: previews[graphqlVersion],
|
||||
upcomingChangesForCurrentVersion: upcomingChanges[graphqlVersion],
|
||||
prerenderedObjectsForCurrentVersion: prerenderedObjects[graphqlVersion],
|
||||
changelog
|
||||
}
|
||||
|
||||
// borrowed from middleware/contextualizers/rest.js
|
||||
context.restGitHubAppsLink = getVersionedPathWithLanguage(
|
||||
'/developers/apps',
|
||||
pageVersion,
|
||||
languageCode
|
||||
)
|
||||
|
||||
context.operationsForCurrentProduct = context.rest.operations[pageVersion] || []
|
||||
|
||||
if (relevantPermalink.href.includes('rest/reference/')) {
|
||||
const docsPath = relevantPermalink.href
|
||||
.split('rest/reference/')[1]
|
||||
.split('#')[0] // do not include #fragments
|
||||
|
||||
// find all operations that with an operationID that matches the requested docs path
|
||||
context.currentRestOperations = context.operationsForCurrentProduct
|
||||
.filter(operation => operation.operationId.startsWith(docsPath))
|
||||
}
|
||||
|
||||
// collect elements of the page that may contain links
|
||||
const pageContent = relevantPermalink.href.includes('graphql/reference/objects')
|
||||
? page.markdown + context.graphql.prerenderedObjectsForCurrentVersion.html
|
||||
: page.intro + page.permissions + page.markdown
|
||||
|
||||
// renderContent is much faster than page.render, even though we later have to run
|
||||
// rewriteLocalLinks in check-images and rewriteAssetPathsToS3 in check-links
|
||||
const pageHtml = await renderContent(pageContent, context)
|
||||
const $ = cheerio.load(pageHtml, { xmlMode: true })
|
||||
|
||||
// check images
|
||||
const { brokenImages: brokenImagesPerVersion, checkedImageCache } = await checkImages($, pageVersion, page.relativePath, checkedImages)
|
||||
if (brokenImagesPerVersion.length) brokenImagesPerPage[pageVersion] = brokenImagesPerVersion
|
||||
checkedImages = checkedImageCache
|
||||
|
||||
// check anchors and links
|
||||
const { brokenLinks: brokenLinksPerVersion, checkedLinkCache } = await checkLinks($, page, context, pageVersion, checkedLinks)
|
||||
if (brokenLinksPerVersion.anchors.length) brokenAnchorsPerPage[pageVersion] = brokenLinksPerVersion.anchors
|
||||
if (brokenLinksPerVersion.links.length) brokenLinksPerPage[pageVersion] = brokenLinksPerVersion.links
|
||||
checkedLinks = checkedLinkCache
|
||||
}
|
||||
|
||||
if (Object.keys(brokenImagesPerPage).length) brokenImages[page.fullPath] = brokenImagesPerPage
|
||||
if (Object.keys(brokenAnchorsPerPage).length) brokenAnchors[page.fullPath] = brokenAnchorsPerPage
|
||||
if (Object.keys(brokenLinksPerPage).length) brokenLinks[page.fullPath] = brokenLinksPerPage
|
||||
}
|
||||
|
||||
done()
|
||||
})
|
||||
|
||||
test('every page has image references that can be resolved', async () => {
|
||||
const numbrokenImages = getNumBrokenItems(brokenImages)
|
||||
expect(numbrokenImages, `Found ${numbrokenImages} total broken images: ${JSON.stringify(brokenImages, null, 2)}`).toBe(0)
|
||||
})
|
||||
|
||||
test.skip('every page has links with anchors that can be resolved', async () => {
|
||||
const numbrokenAnchors = getNumBrokenItems(brokenAnchors)
|
||||
expect(numbrokenAnchors, `Found ${numbrokenAnchors} total broken anchors: ${JSON.stringify(brokenAnchors, null, 2)}`).toBe(0)
|
||||
})
|
||||
|
||||
// disable anchor test til we resolve broken anchors
|
||||
test.skip('every page has links that can be resolved', async () => {
|
||||
const numbrokenLinks = getNumBrokenItems(brokenLinks)
|
||||
expect(numbrokenLinks, `Found ${numbrokenLinks} total broken links: ${JSON.stringify(brokenLinks, null, 2)}`).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
// count all the nested items
|
||||
function getNumBrokenItems (items) {
|
||||
// filter for entries like this:
|
||||
// '/article-path-here.md.dotcom.1.broken link': '/en/articles/foo',
|
||||
return Object.keys(flat(items))
|
||||
.filter(key => last(key.split('.')).includes('broken'))
|
||||
.length
|
||||
}
|
||||
|
|
@ -1,113 +1,47 @@
|
|||
const cheerio = require('cheerio')
|
||||
const { loadPages, loadPageMap } = require('../../lib/pages')
|
||||
const loadSiteData = require('../../lib/site-data')
|
||||
const getApplicableVersions = require('../../lib/get-applicable-versions')
|
||||
const renderContent = require('../../lib/render-content')
|
||||
const checkImages = require('../../lib/check-images')
|
||||
const checkLinks = require('../../lib/check-links')
|
||||
const enterpriseServerVersions = Object.keys(require('../../lib/all-versions'))
|
||||
.filter(version => version.startsWith('enterprise-server@'))
|
||||
const flat = require('flat')
|
||||
const { last } = require('lodash')
|
||||
|
||||
// english only for now
|
||||
const { LinksChecker, buildInitialContext, buildPathContext } = require('../helpers/links-checker')
|
||||
const { uniq } = require('lodash')
|
||||
const languageCode = 'en'
|
||||
|
||||
const context = { currentLanguage: languageCode }
|
||||
|
||||
const loadRedirects = require('../../lib/redirects/precompile')
|
||||
// TODO set to true when we're ready to report and fix broken anchors
|
||||
const checkExternalAnchors = false
|
||||
|
||||
describe('page rendering', () => {
|
||||
jest.setTimeout(1000 * 1000)
|
||||
|
||||
const brokenImages = {}
|
||||
const brokenAnchors = {}
|
||||
const brokenLinks = {}
|
||||
const linksChecker = new LinksChecker()
|
||||
|
||||
beforeAll(async (done) => {
|
||||
const pageList = await loadPages()
|
||||
const pageMap = await loadPageMap(pageList)
|
||||
const siteData = await loadSiteData()
|
||||
const redirects = await loadRedirects(pageList, pageMap)
|
||||
// fetch context.pages, context.redirects, etc.
|
||||
// we only want to build these one time
|
||||
const context = await buildInitialContext()
|
||||
|
||||
context.pages = pageMap
|
||||
context.site = siteData[languageCode].site
|
||||
context.redirects = redirects
|
||||
|
||||
let checkedLinks = {}
|
||||
let checkedImages = {}
|
||||
|
||||
const englishPages = pageList
|
||||
const englishPages = uniq(Object.values(context.pages))
|
||||
.filter(page => page.languageCode === languageCode)
|
||||
// ignore developers content, to be checked separately
|
||||
.filter(page => !page.relativePath.match(/^(rest|graphql|developers)/))
|
||||
|
||||
for (const page of englishPages) {
|
||||
// skip map topics because they have no content of their own
|
||||
if (page.mapTopic) continue
|
||||
|
||||
const brokenImagesPerPage = {}
|
||||
const brokenAnchorsPerPage = {}
|
||||
const brokenLinksPerPage = {}
|
||||
|
||||
// get an array of the pages product versions
|
||||
const pageVersions = getApplicableVersions(page.versions, page.relativePath)
|
||||
|
||||
for (const pageVersion of pageVersions) {
|
||||
// attach page-specific properties to context
|
||||
page.version = pageVersion
|
||||
context.page = page
|
||||
context.currentVersion = pageVersion
|
||||
context.enterpriseServerVersions = enterpriseServerVersions
|
||||
|
||||
// collect elements of the page that may contain links
|
||||
const pageContent = page.intro + page.permissions + page.markdown
|
||||
|
||||
// renderContent is much faster than page.render, even though we later have to run
|
||||
// rewriteLocalLinks in check-images and rewriteAssetPathsToS3 in check-links
|
||||
const pageHtml = await renderContent(pageContent, context)
|
||||
const $ = cheerio.load(pageHtml, { xmlMode: true })
|
||||
|
||||
// check images
|
||||
const { brokenImages: brokenImagesPerVersion, checkedImageCache } = await checkImages($, pageVersion, page.relativePath, checkedImages)
|
||||
if (brokenImagesPerVersion.length) brokenImagesPerPage[pageVersion] = brokenImagesPerVersion
|
||||
checkedImages = checkedImageCache
|
||||
|
||||
// check anchors and links
|
||||
const { brokenLinks: brokenLinksPerVersion, checkedLinkCache } = await checkLinks($, page, context, pageVersion, checkedLinks)
|
||||
if (brokenLinksPerVersion.anchors.length) brokenAnchorsPerPage[pageVersion] = brokenLinksPerVersion.anchors
|
||||
if (brokenLinksPerVersion.links.length) brokenLinksPerPage[pageVersion] = brokenLinksPerVersion.links
|
||||
checkedLinks = checkedLinkCache
|
||||
for (const permalink of page.permalinks) {
|
||||
const pathContext = await buildPathContext(context, page, permalink)
|
||||
await linksChecker.checkPage(pathContext, checkExternalAnchors)
|
||||
}
|
||||
|
||||
if (Object.keys(brokenImagesPerPage).length) brokenImages[page.fullPath] = brokenImagesPerPage
|
||||
if (Object.keys(brokenAnchorsPerPage).length) brokenAnchors[page.fullPath] = brokenAnchorsPerPage
|
||||
if (Object.keys(brokenLinksPerPage).length) brokenLinks[page.fullPath] = brokenLinksPerPage
|
||||
}
|
||||
|
||||
done()
|
||||
})
|
||||
|
||||
test('every page has image references that can be resolved', async () => {
|
||||
const numbrokenImages = getNumBrokenItems(brokenImages)
|
||||
expect(numbrokenImages, `Found ${numbrokenImages} total broken images: ${JSON.stringify(brokenImages, null, 2)}`).toBe(0)
|
||||
const result = await linksChecker.getBrokenImages()
|
||||
expect(result.size, `Found ${result.size} total broken images: ${JSON.stringify([...result], null, 2)}`).toBe(0)
|
||||
})
|
||||
|
||||
test('every page has links with anchors that can be resolved', async () => {
|
||||
const numbrokenAnchors = getNumBrokenItems(brokenAnchors)
|
||||
expect(numbrokenAnchors, `Found ${numbrokenAnchors} total broken anchors: ${JSON.stringify(brokenAnchors, null, 2)}`).toBe(0)
|
||||
// When ready to unskip this,
|
||||
test.skip('every page has links with anchors that can be resolved', async () => {
|
||||
const result = await linksChecker.getBrokenAnchors()
|
||||
const numBrokenAnchors = [...result].reduce((accumulator, [path, anchors]) => accumulator + Object.keys(anchors).length, 0)
|
||||
expect(numBrokenAnchors, `Found ${numBrokenAnchors} total broken anchors in ${result.size} pages: ${JSON.stringify([...result], null, 2)}`).toBe(0)
|
||||
})
|
||||
|
||||
test('every page has links that can be resolved', async () => {
|
||||
const numbrokenLinks = getNumBrokenItems(brokenLinks)
|
||||
expect(numbrokenLinks, `Found ${numbrokenLinks} total broken links: ${JSON.stringify(brokenLinks, null, 2)}`).toBe(0)
|
||||
test('every page has links that can be resolved', () => {
|
||||
const result = linksChecker.getBrokenLinks()
|
||||
expect(result.size, `Found ${result.size} total broken links: ${JSON.stringify([...result], null, 2)}`).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
// count all the nested items
|
||||
function getNumBrokenItems (items) {
|
||||
// filter for entries like this:
|
||||
// '/article-path-here.md.dotcom.1.broken link': '/en/articles/foo',
|
||||
return Object.keys(flat(items))
|
||||
.filter(key => last(key.split('.')).includes('broken'))
|
||||
.length
|
||||
}
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче