зеркало из https://github.com/github/docs.git
Add playwright-axe test suite (#41814)
This commit is contained in:
Hector Alfaro
GitHub
Родитель
bc5f3763cb
Коммит
f3a7ea9d5a
|
|
@ -26,6 +26,14 @@ jobs:
|
|||
playwright-tests:
|
||||
if: github.repository == 'github/docs-internal' || github.repository == 'github/docs'
|
||||
runs-on: ${{ fromJSON('["ubuntu-latest", "ubuntu-20.04-xl"]')[github.repository == 'github/docs-internal'] }}
|
||||
strategy:
|
||||
# When we're comfortable a11y tests aren't generating false positives and helping,
|
||||
# let's remove the matrix and just run playwright in a single job.
|
||||
matrix:
|
||||
node:
|
||||
- playwright-rendering
|
||||
- playwright-a11y
|
||||
fail-fast: false
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Check out repo
|
||||
|
|
@ -46,4 +54,7 @@ jobs:
|
|||
- name: Run Playwright tests
|
||||
env:
|
||||
PLAYWRIGHT_WORKERS: ${{ fromJSON('[1, 4]')[github.repository == 'github/docs-internal'] }}
|
||||
run: npm run playwright-test -- --reporter list
|
||||
# Run playwright rendering tests and a11y tests (axe scans) as distinct checks
|
||||
# so that we can run them without blocking merges until we can be confident
|
||||
# results for a11y tests are meaningul and scenarios we're testing are correct.
|
||||
run: npm run playwright-test -- ${{ matrix.node }} --reporter list
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@
|
|||
"devDependencies": {
|
||||
"@actions/core": "^1.10.0",
|
||||
"@actions/github": "^5.0.3",
|
||||
"@axe-core/playwright": "^4.7.3",
|
||||
"@graphql-inspector/core": "^5.0.0",
|
||||
"@graphql-tools/load": "^8.0.0",
|
||||
"@jest/globals": "29.7.0",
|
||||
|
|
@ -222,6 +223,18 @@
|
|||
"node": ">=6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@axe-core/playwright": {
|
||||
"version": "4.7.3",
|
||||
"resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.7.3.tgz",
|
||||
"integrity": "sha512-v2PRgAyGvop7bamrTpNJtc5b1R7giAPnMzZXrS/VDZBCY5+uwVYtCNgDvBsqp5P1QMZxUMoBN+CERJUTMjFN0A==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"axe-core": "^4.7.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"playwright-core": ">= 1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/code-frame": {
|
||||
"version": "7.18.6",
|
||||
"license": "MIT",
|
||||
|
|
|
|||
|
|
@ -144,6 +144,7 @@
|
|||
"devDependencies": {
|
||||
"@actions/core": "^1.10.0",
|
||||
"@actions/github": "^5.0.3",
|
||||
"@axe-core/playwright": "^4.7.3",
|
||||
"@graphql-inspector/core": "^5.0.0",
|
||||
"@graphql-tools/load": "^8.0.0",
|
||||
"@jest/globals": "29.7.0",
|
||||
|
|
|
|||
|
|
@ -56,12 +56,15 @@ export const SidebarProduct = () => {
|
|||
return (
|
||||
<>
|
||||
<div className="ml-3">
|
||||
<NavList aria-label="REST sidebar">
|
||||
<NavList aria-label="REST sidebar overview articles">
|
||||
{conceptualPages.map((childPage) => (
|
||||
<NavListItem key={childPage.href} childPage={childPage} />
|
||||
))}
|
||||
</NavList>
|
||||
|
||||
<hr data-testid="rest-sidebar-reference" />
|
||||
<hr data-testid="rest-sidebar-reference" className="m-2" />
|
||||
|
||||
<NavList aria-label="REST sidebar reference pages">
|
||||
{restPages.map((category) => (
|
||||
<RestNavListItem key={category.href} category={category} />
|
||||
))}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ export const TableOfContents = (props: Props) => {
|
|||
const { items, variant = 'expanded' } = props
|
||||
|
||||
return (
|
||||
<ul
|
||||
<div
|
||||
data-testid="table-of-contents"
|
||||
className={cx(variant === 'compact' ? 'list-style-outside pl-2' : '')}
|
||||
>
|
||||
|
|
@ -22,7 +22,7 @@ export const TableOfContents = (props: Props) => {
|
|||
const { fullPath: href, title, intro } = item
|
||||
|
||||
return (
|
||||
<li
|
||||
<div
|
||||
key={href}
|
||||
data-testid="expanded-item"
|
||||
className="pt-4 pb-3 f4 d-list-item width-full list-style-none border-bottom"
|
||||
|
|
@ -35,7 +35,7 @@ export const TableOfContents = (props: Props) => {
|
|||
{intro && (
|
||||
<div className="f4 color-fg-muted" dangerouslySetInnerHTML={{ __html: intro }} />
|
||||
)}
|
||||
</li>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
|
||||
|
|
@ -52,28 +52,30 @@ export const TableOfContents = (props: Props) => {
|
|||
<Link href={href}>{title}</Link>
|
||||
</ActionList.Item>
|
||||
{(childTocItems || []).length > 0 && (
|
||||
<ul
|
||||
className={cx(
|
||||
variant === 'compact' ? 'list-style-circle pl-5' : 'list-style-none',
|
||||
)}
|
||||
>
|
||||
{(childTocItems || []).filter(Boolean).map((childItem) => {
|
||||
return (
|
||||
<ActionList.Item
|
||||
key={childItem.fullPath}
|
||||
className="f4 color-fg-accent d-list-item d-block width-full text-underline"
|
||||
>
|
||||
<Link href={childItem.fullPath}>{childItem.title}</Link>
|
||||
</ActionList.Item>
|
||||
)
|
||||
})}
|
||||
</ul>
|
||||
<li className="f4 color-fg-accent d-list-item d-block width-full text-underline">
|
||||
<ActionList
|
||||
className={cx(
|
||||
variant === 'compact' ? 'list-style-circle pl-5' : 'list-style-none',
|
||||
)}
|
||||
>
|
||||
{(childTocItems || []).filter(Boolean).map((childItem) => {
|
||||
return (
|
||||
<ActionList.Item
|
||||
key={childItem.fullPath}
|
||||
className="f4 color-fg-accent d-list-item d-block width-full text-underline"
|
||||
>
|
||||
<Link href={childItem.fullPath}>{childItem.title}</Link>
|
||||
</ActionList.Item>
|
||||
)
|
||||
})}
|
||||
</ActionList>
|
||||
</li>
|
||||
)}
|
||||
</React.Fragment>
|
||||
)
|
||||
})}
|
||||
</ActionList>
|
||||
)}
|
||||
</ul>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -248,7 +248,7 @@ export function RestCodeSamples({ operation, slug, heading }: Props) {
|
|||
</div>
|
||||
<div className="border-top d-inline-flex flex-justify-between width-full flex-items-center pt-2">
|
||||
<div className="d-inline-flex ml-2">
|
||||
<TabNav aria-label="Example language selector">
|
||||
<TabNav aria-label={`Example language selector for ${operation.title}`}>
|
||||
{languageSelectOptions.map((optionKey) => (
|
||||
<TabNav.Link
|
||||
key={optionKey}
|
||||
|
|
@ -300,6 +300,8 @@ export function RestCodeSamples({ operation, slug, heading }: Props) {
|
|||
`border-top rounded-1 my-0 ${getLanguageHighlight(selectedLanguage)}`,
|
||||
)}
|
||||
data-highlight={getLanguageHighlight(selectedLanguage)}
|
||||
// eslint-disable-next-line jsx-a11y/no-noninteractive-tabindex
|
||||
tabIndex={0}
|
||||
>
|
||||
<code ref={requestCodeExample}>{displayedExample[selectedLanguage]}</code>
|
||||
</div>
|
||||
|
|
@ -314,7 +316,10 @@ export function RestCodeSamples({ operation, slug, heading }: Props) {
|
|||
></h4>
|
||||
<div className="border rounded-1">
|
||||
{displayedExample.response.schema ? (
|
||||
<TabNav className="pt-2 mx-2" aria-label="Example response format selector">
|
||||
<TabNav
|
||||
className="pt-2 mx-2"
|
||||
aria-label={`Example response format selector for ${operation.title}`}
|
||||
>
|
||||
{responseSelectOptions.map((optionKey) => (
|
||||
<TabNav.Link
|
||||
key={optionKey}
|
||||
|
|
@ -353,6 +358,8 @@ export function RestCodeSamples({ operation, slug, heading }: Props) {
|
|||
)}
|
||||
data-highlight={'json'}
|
||||
style={{ maxHeight: responseMaxHeight }}
|
||||
// eslint-disable-next-line jsx-a11y/no-noninteractive-tabindex
|
||||
tabIndex={0}
|
||||
>
|
||||
<code ref={responseCodeExample}>
|
||||
{selectedResponse === ResponseKeys.example
|
||||
|
|
|
|||
|
|
@ -0,0 +1,84 @@
|
|||
# Axe test suite
|
||||
We're using Playwright with the Axe Accessibility Engine to automatically flag accessibility issues with our site. This document summarizes some of the behavior we're enabling, and provides you with techniques and supporting documentation for working with this tooling.
|
||||
|
||||
**Please note**: accessibility testing requires both automated and manual testing. The results from this tool are not comprehensive, and they need to be validated against best practices and our own accessibility standards. Furthermore, at the time of this writing, we're not testing every single page or interactive element on our site, so there are plenty of opportunities to improve upon this implementation and we've got lots of room to mature.
|
||||
|
||||
## What we test
|
||||
In this first iteration of these tests, we're testing some [high-usage pages, layouts, and elements](https://github.com/github/docs-internal/blob/30ff2bd725d3dfc326da3ed2f90fe8e87a065c93/tests/rendering-fixtures/playwright-a11y.spec.ts#L5-L21) that are at risk of becoming accessibility concerns. We understand the risk relatively well because we've fixed these types of accessibility issues in the past, but the test suite attempts to avoid making the same mistakes.
|
||||
|
||||
The Axe tests aren't comprehensive; for example, when elements are hidden in the DOM they may be missed. Over time, we should create more fine-grained tests that manipulate interactive elements we know can result in accessibility issues.
|
||||
|
||||
## How to interpret test results
|
||||
A results from the Axe test suite will be verbose. We've highlighted some fields you'll find in the results below.
|
||||
|
||||
```shell
|
||||
# Example 1
|
||||
+ "description": "Ensures table headers have discernible text",
|
||||
+ "help": "Table header text should not be empty",
|
||||
+ "helpUrl": "https://dequeuniversity.com/rules/axe/4.7/empty-table-header?application=playwright",
|
||||
+ "id": "empty-table-header",
|
||||
+ "impact": "minor",
|
||||
|
||||
# Example 2
|
||||
+ "description": "Ensures that lists are structured correctly",
|
||||
+ "help": "<ul> and <ol> must only directly contain <li>, <script> or <template> elements",
|
||||
+ "helpUrl": "https://dequeuniversity.com/rules/axe/4.7/list?application=playwright",
|
||||
+ "id": "list",
|
||||
+ "impact": "serious",
|
||||
```
|
||||
- `description`: explains the rule that returned an error
|
||||
- `help`: explains the expected result from the rule
|
||||
- `helpUrl`: links to a short page on Deque University (the makers of Axe) which will usually contain examples of both valid and invalid markup. Note that Deque University will link a lot of premium content from this page that will not be accessible without paying, so you may need to use this information as a jumping off point in doing your own research
|
||||
- `id`: an identifier for the rule, can usually be cross-referenced with [Axe's HTML rules](https://dequeuniversity.com/rules/axe/4.7)
|
||||
- `impact`: the impact on a user
|
||||
|
||||
```shell
|
||||
# Example 3
|
||||
+ "message": "Element does not have text that is visible to screen readers",
|
||||
+ "failureSummary": "Fix any of the following:
|
||||
+ Element does not have text that is visible to screen readers",
|
||||
+ "html": "<th scope=\"col\"></th>",
|
||||
+ "impact": "minor",
|
||||
|
||||
# Example 4
|
||||
+ "failureSummary": "Fix all of the following:
|
||||
+ List element has direct children that are not allowed: hr",
|
||||
+ "html": "<ul class=\"List__ListBox-sc-1x7olzq-0 hgjakc\">",
|
||||
+ "impact": "serious",
|
||||
```
|
||||
- `message`: more specific information that'll help you troubleshoot the failure
|
||||
- `failureSummary`: a hint on how you could fix the problem
|
||||
|
||||
```shell
|
||||
# Example 5
|
||||
+ "target": Array [
|
||||
+ ".ghd-tool > table > thead > tr > th[scope=\"col\"]:nth-child(1)",
|
||||
+ ]
|
||||
+ "tags": Array [
|
||||
+ "cat.name-role-value",
|
||||
+ "best-practice",
|
||||
+ ],
|
||||
|
||||
# Example 6
|
||||
+ "target": Array [
|
||||
+ ".List__ListBox-sc-1x7olzq-0",
|
||||
+ ],
|
||||
+ "tags": Array [
|
||||
+ "cat.structure",
|
||||
+ "wcag2a",
|
||||
+ "wcag131",
|
||||
+ ],
|
||||
```
|
||||
- `target`: selector for the elements containing a failure
|
||||
- `tags`: information about the severity or identifiers for the [WCAG success criteria](https://www.w3.org/TR/WCAG21/)
|
||||
|
||||
## Why we're doing this
|
||||
We'd like our site to as accessible as possible.
|
||||
|
||||
Until recently, we've reacted to accessibility audits. In an audit, third-party auditors manually test specific pages from our site following a list of scenarios we provide them. Findings are reported through audit issues which require us to fix them within a designated timeframe. This process taught us a lot about our shortcomings and we now have the opportunity to be proactive.
|
||||
|
||||
[Axe DevTools](https://chrome.google.com/webstore/detail/axe-devtools-web-accessib/lhdoppojpmngadmnindnejefpokejbdd) is a tool that our auditors use for a majority -- not all -- of their testing. We can run the same engine with Playwright, which we already use. So we now have the ability to flag and fix potential accessibility issues *before* they're found in an audit.
|
||||
|
||||
## Supporting documentation
|
||||
- [Accessibility testing](https://playwright.dev/docs/accessibility-testing) on Playwright Docs
|
||||
- [@axe-core/playwright](https://github.com/dequelabs/axe-core-npm/blob/764ad64102d7b6b5186c5a4e1d79fff190991740/packages/playwright/README.md) NPM package, API documentation on GitHub
|
||||
|
|
@ -18,5 +18,6 @@ children:
|
|||
- /ifversion
|
||||
- /links-with-liquid
|
||||
- /tool-specific
|
||||
- /tool-platform-switcher
|
||||
- /data
|
||||
---
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ type: how_to
|
|||
|
||||
{% rowheaders %}
|
||||
|
||||
| |Tom|Dick|Harry|
|
||||
|Preferences|Tom|Dick|Harry|
|
||||
|---|---|---|---|
|
||||
|Favorite fruit| Banana | Apple | Mango
|
||||
|Football or soccer|Soccer|Football|Sportsball??
|
||||
|
|
@ -22,8 +22,8 @@ type: how_to
|
|||
|
||||
## Regular table
|
||||
|
||||
| Name | Seasons | Skill
|
||||
|---|---|---|---|
|
||||
| Name | Seasons | Skill |
|
||||
|---|---|---|
|
||||
| Jill | 1 | Nunchucks |
|
||||
| Sabrina | 3 | Knives |
|
||||
| Kelly | 4 | Kicks |
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
---
|
||||
title: Tool Platform switcher
|
||||
intro: Combines tool and platform switchers
|
||||
defaultTool: desktop
|
||||
defaultPlatform: mac
|
||||
versions:
|
||||
fpt: '*'
|
||||
ghes: '*'
|
||||
ghae: '*'
|
||||
ghec: '*'
|
||||
type: tutorial
|
||||
---
|
||||
|
||||
## Running a workflow
|
||||
|
||||
This page has a tool switcher
|
||||
|
||||
{% webui %}
|
||||
|
||||
1. this is webui content
|
||||
|
||||
{% endwebui %}
|
||||
|
||||
{% cli %}
|
||||
|
||||
this is cli content
|
||||
|
||||
```shell
|
||||
cli content
|
||||
```
|
||||
|
||||
{% endcli %}
|
||||
|
||||
{% desktop %}
|
||||
this is desktop content
|
||||
{% enddesktop %}
|
||||
|
||||
{% webui %}
|
||||
|
||||
## Webui section
|
||||
|
||||
Webui section specific content
|
||||
|
||||
{% endwebui %}
|
||||
|
||||
{% desktop %}
|
||||
|
||||
## Desktop section
|
||||
|
||||
Desktop section specific content
|
||||
|
||||
{% enddesktop %}
|
||||
|
||||
## General
|
||||
|
||||
Bla bla
|
||||
|
||||
{% mac %}
|
||||
|
||||
## Macintosh until 1999
|
||||
|
||||
`mac` specific content.
|
||||
|
||||
{% endmac %}
|
||||
|
||||
{% windows %}
|
||||
|
||||
## Windows 95 was awesome
|
||||
|
||||
`windows` specific content.
|
||||
|
||||
{% endwindows %}
|
||||
|
||||
{% linux %}
|
||||
|
||||
## The year of Linux on the desktop
|
||||
|
||||
`linux` specific content.
|
||||
|
||||
{% endlinux %}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
---
|
||||
title: Code annotations
|
||||
intro: This page has code annotations
|
||||
versions:
|
||||
fpt: '*'
|
||||
ghes: '*'
|
||||
ghae: '*'
|
||||
ghec: '*'
|
||||
type: how_to
|
||||
layout: inline
|
||||
---
|
||||
|
||||
## Introduction
|
||||
|
||||
This page is about testing code annotations. It needs some content...
|
||||
|
||||
Saepe nam voluptatem nulla tempora est voluptate neque et. Quis quae iste corporis. Veritatis reprehenderit ut in labore.
|
||||
|
||||
## The example
|
||||
|
||||
Ad explicabo fuga iusto hic et. Officiis ratione delectus atque molestiae sint et error porro. Modi consequatur tenetur sequi consequatur et. Et est quaerat facilis amet amet excepturi nesciunt deserunt.
|
||||
At esse delectus repudiandae ducimus. Voluptas architecto enim et. Repellendus explicabo omnis nostrum tenetur cumque rem qui.
|
||||
|
||||
```javascript annotate
|
||||
// Greet the world!
|
||||
console.log("Hello, world!")
|
||||
|
||||
// What about Saturn?
|
||||
console.log("Hello, Saturn!")
|
||||
```
|
||||
## And the conclusion
|
||||
|
||||
Minima non repellat molestiae non placeat natus. Quo vitae esse et quo. Atque neque voluptatem qui voluptas sapiente quae. Est et animi aut repellendus qui aperiam ullam. Soluta laboriosam voluptatem ipsam recusandae voluptate et aut.
|
||||
Quaerat culpa nulla itaque cum doloremque consequatur. Maxime quia dolor ullam eligendi magnam distinctio quae reiciendis. Natus ea perferendis et odio.
|
||||
|
|
@ -9,4 +9,5 @@ versions:
|
|||
children:
|
||||
- /intro
|
||||
- /permissions
|
||||
- /code-annotations
|
||||
---
|
||||
|
|
|
|||
|
|
@ -128,7 +128,7 @@ describe('rowheaders', () => {
|
|||
// (and there are 4 of these <tr> rows)
|
||||
const secondTable = tables.filter((i) => i === 1)
|
||||
expect($('tbody tr th', secondTable).length).toBe(0)
|
||||
expect($('tbody tr td', secondTable).length).toBe(3 * 4)
|
||||
expect($('tbody tr td', secondTable).length).toBe(3 * 3)
|
||||
|
||||
// More specifically, the <th> tags should have the appropriate
|
||||
// `scope` attribute.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
import { test, expect } from '@playwright/test'
|
||||
import AxeBuilder from '@axe-core/playwright'
|
||||
|
||||
const pages: { [key: string]: string } = {
|
||||
category: '/actions/category',
|
||||
codeAnnotations: '/get-started/markdown/code-annotations',
|
||||
guides: '/code-security/guides',
|
||||
homepage: '/',
|
||||
learningPath:
|
||||
'/code-security/getting-started/quickstart?learn=foo_bar&learnProduct=code-security',
|
||||
mapAndTopic: '/actions/category/map-topic',
|
||||
procedural: '/get-started/images/images-in-lists',
|
||||
productLanding: '/code-security',
|
||||
restCategory: '/rest/actions/artifacts',
|
||||
restLanding: '/rest',
|
||||
restOverview: '/rest/overview/about-githubs-apis',
|
||||
search: '/search?q=playwright',
|
||||
switchers: '/get-started/liquid/tool-platform-switcher',
|
||||
tableWithHeaders: '/get-started/liquid/table-row-headers',
|
||||
video: '/get-started',
|
||||
videoTranscript: '/get-started/video-transcripts/transcript--my-awesome-video',
|
||||
}
|
||||
|
||||
// create a test for each page, will eventually be separated into finer grain tests
|
||||
Object.keys(pages).forEach((pageName) => {
|
||||
test.describe(`${pageName}`, () => {
|
||||
test('full page axe scan', async ({ page }) => {
|
||||
await page.goto(pages[pageName])
|
||||
|
||||
const accessibilityScanResults = await new AxeBuilder({ page }).analyze()
|
||||
|
||||
expect(accessibilityScanResults.violations).toEqual([])
|
||||
})
|
||||
})
|
||||
})
|
||||
Загрузка…
Ссылка в новой задаче