зеркало из https://github.com/github/docs.git
Translation cleanup (#33738)
This commit is contained in:
Kevin Heis
GitHub
Родитель
d8f706be32
Коммит
83af1896b8
|
|
@ -1 +0,0 @@
|
|||
ALLOW_TRANSLATION_COMMITS=
|
||||
|
|
@ -1,142 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import fs from 'fs'
|
||||
import github from '@actions/github'
|
||||
|
||||
const OPTIONS = Object.fromEntries(
|
||||
['BASE', 'BODY_FILE', 'GITHUB_TOKEN', 'HEAD', 'LANGUAGE', 'TITLE', 'GITHUB_REPOSITORY'].map(
|
||||
(envVarName) => {
|
||||
const envVarValue = process.env[envVarName]
|
||||
if (!envVarValue) {
|
||||
throw new Error(`You must supply a ${envVarName} environment variable`)
|
||||
}
|
||||
return [envVarName, envVarValue]
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
if (!process.env.GITHUB_REPOSITORY) {
|
||||
throw new Error('GITHUB_REPOSITORY environment variable not set')
|
||||
}
|
||||
|
||||
const RETRY_STATUSES = [
|
||||
422, // Retry the operation if the PR already exists
|
||||
502, // Retry the operation if the API responds with a `502 Bad Gateway` error.
|
||||
]
|
||||
const RETRY_ATTEMPTS = 3
|
||||
const {
|
||||
// One of the default environment variables provided by Actions.
|
||||
GITHUB_REPOSITORY,
|
||||
|
||||
// These are passed in from the step in the workflow file.
|
||||
TITLE,
|
||||
BASE,
|
||||
HEAD,
|
||||
LANGUAGE,
|
||||
BODY_FILE,
|
||||
GITHUB_TOKEN,
|
||||
} = OPTIONS
|
||||
const [OWNER, REPO] = GITHUB_REPOSITORY.split('/')
|
||||
|
||||
const octokit = github.getOctokit(GITHUB_TOKEN)
|
||||
|
||||
/**
|
||||
* @param {object} config Configuration options for finding the PR.
|
||||
* @returns {Promise<number | undefined>} The PR number.
|
||||
*/
|
||||
async function findPullRequestNumber(config) {
|
||||
// Get a list of PRs and see if one already exists.
|
||||
const { data: listOfPullRequests } = await octokit.rest.pulls.list({
|
||||
owner: config.owner,
|
||||
repo: config.repo,
|
||||
head: `${config.owner}:${config.head}`,
|
||||
})
|
||||
|
||||
return listOfPullRequests[0]?.number
|
||||
}
|
||||
|
||||
/**
|
||||
* When this file was first created, we only introduced support for creating a pull request for some translation batch.
|
||||
* However, some of our first workflow runs failed during the pull request creation due to a timeout error.
|
||||
* There have been cases where, despite the timeout error, the pull request gets created _anyway_.
|
||||
* To accommodate this reality, we created this function to look for an existing pull request before a new one is created.
|
||||
* Although the "find" check is redundant in the first "cycle", it's designed this way to recursively call the function again via its retry mechanism should that be necessary.
|
||||
*
|
||||
* @param {object} config Configuration options for creating the pull request.
|
||||
* @returns {Promise<number>} The PR number.
|
||||
*/
|
||||
async function findOrCreatePullRequest(config) {
|
||||
const found = await findPullRequestNumber(config)
|
||||
|
||||
if (found) {
|
||||
return found
|
||||
}
|
||||
|
||||
try {
|
||||
const { data: pullRequest } = await octokit.rest.pulls.create({
|
||||
owner: config.owner,
|
||||
repo: config.repo,
|
||||
base: config.base,
|
||||
head: config.head,
|
||||
title: config.title,
|
||||
body: config.body,
|
||||
draft: false,
|
||||
})
|
||||
|
||||
return pullRequest.number
|
||||
} catch (error) {
|
||||
if (!error.response || !config.retryCount) {
|
||||
throw error
|
||||
}
|
||||
|
||||
if (!config.retryStatuses.includes(error.response.status)) {
|
||||
throw error
|
||||
}
|
||||
|
||||
console.error(`Error creating pull request: ${error.message}`)
|
||||
console.warn(`Retrying in 5 seconds...`)
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000))
|
||||
|
||||
config.retryCount -= 1
|
||||
|
||||
return findOrCreatePullRequest(config)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {object} config Configuration options for labeling the PR
|
||||
* @returns {Promise<undefined>}
|
||||
*/
|
||||
async function labelPullRequest(config) {
|
||||
await octokit.rest.issues.update({
|
||||
owner: config.owner,
|
||||
repo: config.repo,
|
||||
issue_number: config.issue_number,
|
||||
labels: config.labels,
|
||||
})
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = {
|
||||
title: TITLE,
|
||||
base: BASE,
|
||||
head: HEAD,
|
||||
body: fs.readFileSync(BODY_FILE, 'utf8'),
|
||||
labels: ['translation-batch', `translation-batch-${LANGUAGE}`],
|
||||
owner: OWNER,
|
||||
repo: REPO,
|
||||
retryStatuses: RETRY_STATUSES,
|
||||
retryCount: RETRY_ATTEMPTS,
|
||||
}
|
||||
|
||||
options.issue_number = await findOrCreatePullRequest(options)
|
||||
const pr = `${GITHUB_REPOSITORY}#${options.issue_number}`
|
||||
console.log(`Created PR ${pr}`)
|
||||
|
||||
// metadata parameters aren't currently available in `github.rest.pulls.create`,
|
||||
// but they are in `github.rest.issues.update`.
|
||||
await labelPullRequest(options)
|
||||
console.log(`Updated ${pr} with these labels: ${options.labels.join(', ')}`)
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -1,193 +0,0 @@
|
|||
name: Create translation Batch Pull Request (Microsoft)
|
||||
|
||||
# **What it does**:
|
||||
# - Creates one pull request per language after running a series of automated checks,
|
||||
# removing translations that are broken in any known way
|
||||
# **Why we have it**:
|
||||
# - To deploy translations
|
||||
# **Who does it impact**: It automates what would otherwise be manual work,
|
||||
# helping docs engineering focus on higher value work
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '20 16 * * *' # Run every day at 16:20 UTC / 8:20 PST
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
create-translation-batch:
|
||||
name: Create translation batch
|
||||
if: github.repository == 'github/docs-internal'
|
||||
runs-on: ubuntu-latest
|
||||
# A sync's average run time is ~3.2 hours.
|
||||
# This sets a maximum execution time of 300 minutes (5 hours) to prevent the workflow from running longer than necessary.
|
||||
timeout-minutes: 300
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
include:
|
||||
- language: es
|
||||
language_dir: translations/es-ES
|
||||
language_repo: github/docs-internal.es-es
|
||||
|
||||
- language: ja
|
||||
language_dir: translations/ja-JP
|
||||
language_repo: github/docs-internal.ja-jp
|
||||
|
||||
- language: pt
|
||||
language_dir: translations/pt-BR
|
||||
language_repo: github/docs-internal.pt-br
|
||||
|
||||
- language: zh
|
||||
language_dir: translations/zh-CN
|
||||
language_repo: github/docs-internal.zh-cn
|
||||
|
||||
# We'll be ready to add the following languages in a future effort.
|
||||
|
||||
- language: ru
|
||||
language_dir: translations/ru-RU
|
||||
language_repo: github/docs-internal.ru-ru
|
||||
|
||||
- language: ko
|
||||
language_dir: translations/ko-KR
|
||||
language_repo: github/docs-internal.ko-kr
|
||||
|
||||
- language: fr
|
||||
language_dir: translations/fr-FR
|
||||
language_repo: github/docs-internal.fr-fr
|
||||
|
||||
- language: de
|
||||
language_dir: translations/de-DE
|
||||
language_repo: github/docs-internal.de-de
|
||||
|
||||
steps:
|
||||
- name: Set branch name
|
||||
id: set-branch
|
||||
run: |
|
||||
echo "BRANCH_NAME=msft-translation-batch-${{ matrix.language }}-$(date +%Y-%m-%d__%H-%M)" >> $GITHUB_OUTPUT
|
||||
- run: git config --global user.name "docubot"
|
||||
- run: git config --global user.email "67483024+docubot@users.noreply.github.com"
|
||||
|
||||
- name: Checkout the docs-internal repo
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
|
||||
- name: Create a branch for the current language
|
||||
run: git checkout -b ${{ steps.set-branch.outputs.BRANCH_NAME }}
|
||||
|
||||
- name: Remove unwanted git hooks
|
||||
run: rm .git/hooks/post-checkout
|
||||
|
||||
- name: Remove all language translations
|
||||
run: |
|
||||
git rm -rf --quiet ${{ matrix.language_dir }}/content
|
||||
git rm -rf --quiet ${{ matrix.language_dir }}/data
|
||||
|
||||
- name: Checkout the language-specific repo
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
|
||||
with:
|
||||
repository: ${{ matrix.language_repo }}
|
||||
token: ${{ secrets.DOCUBOT_READORG_REPO_WORKFLOW_SCOPES }}
|
||||
path: ${{ matrix.language_dir }}
|
||||
|
||||
- name: Remove .git from the language-specific repo
|
||||
run: rm -rf ${{ matrix.language_dir }}/.git
|
||||
|
||||
- name: Commit translated files
|
||||
run: |
|
||||
git add ${{ matrix.language_dir }}
|
||||
git commit -m "Add translations" || echo "Nothing to commit"
|
||||
|
||||
- name: 'Setup node'
|
||||
uses: actions/setup-node@8c91899e586c5b171469028077307d293428b516
|
||||
with:
|
||||
node-version: '16.17.0'
|
||||
|
||||
- run: npm ci
|
||||
|
||||
- name: Homogenize frontmatter
|
||||
run: |
|
||||
node script/i18n/homogenize-frontmatter.js
|
||||
git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/homogenize-frontmatter.js" || echo "Nothing to commit"
|
||||
|
||||
- name: Fix translation errors
|
||||
run: |
|
||||
node script/i18n/fix-translation-errors.js
|
||||
git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/fix-translation-errors.js" || echo "Nothing to commit"
|
||||
|
||||
- name: Check rendering
|
||||
run: |
|
||||
node script/i18n/lint-translation-files.js --check rendering | tee -a /tmp/batch.log | cat
|
||||
git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/lint-translation-files.js --check rendering" || echo "Nothing to commit"
|
||||
|
||||
- name: Reset files with broken liquid tags
|
||||
run: |
|
||||
node script/i18n/msft-reset-files-with-broken-liquid-tags.js --language=${{ matrix.language }} | tee -a /tmp/batch.log | cat
|
||||
git add ${{ matrix.language_dir }} && git commit -m "run script/i18n/msft-reset-files-with-broken-liquid-tags.js --language=${{ matrix.language }}" || echo "Nothing to commit"
|
||||
|
||||
- name: Check in CSV report
|
||||
run: |
|
||||
mkdir -p translations/log
|
||||
csvFile=translations/log/msft-${{ matrix.language }}-resets.csv
|
||||
script/i18n/msft-report-reset-files.js --report-type=csv --language=${{ matrix.language }} --log-file=/tmp/batch.log > $csvFile
|
||||
git add -f $csvFile && git commit -m "Check in ${{ matrix.language }} CSV report" || echo "Nothing to commit"
|
||||
|
||||
- name: Write the reported files that were reset to /tmp/pr-body.txt
|
||||
run: script/i18n/msft-report-reset-files.js --report-type=pull-request-body --language=${{ matrix.language }} --log-file=/tmp/batch.log --csv-path=${{ steps.set-branch.outputs.BRANCH_NAME }}/translations/log/msft-${{ matrix.language }}-resets.csv > /tmp/pr-body.txt
|
||||
|
||||
- name: Push filtered translations
|
||||
run: git push origin ${{ steps.set-branch.outputs.BRANCH_NAME }}
|
||||
|
||||
- name: Close existing stale batches
|
||||
uses: lee-dohm/close-matching-issues@e9e43aad2fa6f06a058cedfd8fb975fd93b56d8f
|
||||
with:
|
||||
token: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }}
|
||||
query: 'type:pr label:translation-batch-${{ matrix.language }}'
|
||||
|
||||
- name: Create translation batch pull request
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.DOCUBOT_REPO_PAT }}
|
||||
TITLE: 'New translation batch for ${{ matrix.language }}'
|
||||
BASE: 'main'
|
||||
HEAD: ${{ steps.set-branch.outputs.BRANCH_NAME }}
|
||||
LANGUAGE: ${{ matrix.language }}
|
||||
BODY_FILE: '/tmp/pr-body.txt'
|
||||
run: .github/actions-scripts/msft-create-translation-batch-pr.js
|
||||
|
||||
- name: Approve PR
|
||||
if: github.ref_name == 'main'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }}
|
||||
run: gh pr review --approve || echo "Nothing to approve"
|
||||
|
||||
- name: Set auto-merge
|
||||
if: github.ref_name == 'main'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }}
|
||||
run: gh pr merge ${{ steps.set-branch.outputs.BRANCH_NAME }} --auto --squash || echo "Nothing to merge"
|
||||
|
||||
# When the maximum execution time is reached for this job, Actions cancels the workflow run.
|
||||
# This emits a notification for the first responder to triage.
|
||||
- name: Send Slack notification if workflow is cancelled
|
||||
uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340
|
||||
if: cancelled()
|
||||
with:
|
||||
channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
|
||||
bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}🎉
|
||||
color: failure
|
||||
text: 'The new translation batch for ${{ matrix.language }} was cancelled.'
|
||||
|
||||
# Emit a notification for the first responder to triage if the workflow failed.
|
||||
- name: Send Slack notification if workflow failed
|
||||
uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340
|
||||
if: failure()
|
||||
with:
|
||||
channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
|
||||
bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
|
||||
color: failure
|
||||
text: 'The new translation batch for ${{ matrix.language }} failed.'
|
||||
|
|
@ -20,7 +20,6 @@ on:
|
|||
- 'lib/webhooks/**'
|
||||
- 'package*.json'
|
||||
- 'script/**'
|
||||
- 'translations/**'
|
||||
- 'content/actions/deployment/security-hardening-your-deployments/**'
|
||||
|
||||
permissions:
|
||||
|
|
@ -49,8 +48,6 @@ jobs:
|
|||
|
||||
# Returns list of changed files matching each filter
|
||||
filters: |
|
||||
translation:
|
||||
- 'translations/**'
|
||||
openapi:
|
||||
- 'lib/rest/static/**'
|
||||
notAllowed:
|
||||
|
|
@ -67,7 +64,6 @@ jobs:
|
|||
- 'lib/webhooks/**'
|
||||
- 'package*.json'
|
||||
- 'scripts/**'
|
||||
- 'translations/**'
|
||||
- 'content/actions/deployment/security-hardening-your-deployments/**'
|
||||
|
||||
# When there are changes to files we can't accept, leave a comment
|
||||
|
|
@ -91,7 +87,6 @@ jobs:
|
|||
'lib/webhooks/**',
|
||||
'package*.json',
|
||||
'scripts/**',
|
||||
'translations/**',
|
||||
'content/actions/deployment/security-hardening-your-deployments/**',
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
/translations/
|
||||
includes/
|
||||
data/release-notes/
|
||||
script/bookmarklets/
|
||||
|
|
|
|||
|
|
@ -1,7 +1,4 @@
|
|||
{
|
||||
"files.exclude": {
|
||||
"translations/**": true
|
||||
},
|
||||
"workbench.editor.enablePreview": false,
|
||||
"workbench.editor.enablePreviewFromQuickOpen": false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
"ignore": [
|
||||
"assets",
|
||||
"script",
|
||||
"translations",
|
||||
"stylesheets",
|
||||
"tests",
|
||||
"content",
|
||||
|
|
|
|||
|
|
@ -451,68 +451,6 @@ A helper that returns an array of files for a given path and file extension.
|
|||
---
|
||||
|
||||
|
||||
### [`i18n/fix-translation-errors.js`](i18n/fix-translation-errors.js)
|
||||
|
||||
Run this script to fix known frontmatter errors by copying values from english file Currently only fixing errors in: 'type', 'changelog' Please double check the changes created by this script before committing.
|
||||
|
||||
---
|
||||
|
||||
|
||||
### [`i18n/homogenize-frontmatter.js`](i18n/homogenize-frontmatter.js)
|
||||
|
||||
Run this script to fix known frontmatter errors by copying values from english file Translatable properties are designated in the frontmatter JSON schema
|
||||
|
||||
---
|
||||
|
||||
|
||||
### [`i18n/lint-translation-files.js`](i18n/lint-translation-files.js)
|
||||
|
||||
Use this script as part of the translation merge process to output a list of either parsing or rendering errors in translated files and run script/i18n/reset-translated-file.js on them.
|
||||
|
||||
---
|
||||
|
||||
|
||||
### [`i18n/msft-report-reset-files.js`](i18n/msft-report-reset-files.js)
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
|
||||
### [`i18n/msft-reset-files-with-broken-liquid-tags.js`](i18n/msft-reset-files-with-broken-liquid-tags.js)
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
|
||||
### [`i18n/msft-tokens.js`](i18n/msft-tokens.js)
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
|
||||
### [`i18n/prune-stale-files.js`](i18n/prune-stale-files.js)
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
|
||||
### [`i18n/reset-translated-file.js`](i18n/reset-translated-file.js)
|
||||
|
||||
This is a convenience script for replacing the contents of translated files with the English content from their corresponding source file.
|
||||
|
||||
Usage: script/i18n/reset-translated-file.js <filename>
|
||||
|
||||
Examples:
|
||||
|
||||
$ script/i18n/reset-translated-file.js translations/es-XL/content/actions/index.md
|
||||
|
||||
---
|
||||
|
||||
|
||||
### [`i18n/test-html-pages.js`](i18n/test-html-pages.js)
|
||||
|
||||
|
||||
|
|
@ -520,13 +458,6 @@ $ script/i18n/reset-translated-file.js translations/es-XL/content/actions/index.
|
|||
---
|
||||
|
||||
|
||||
### [`i18n/test-render-translation.js`](i18n/test-render-translation.js)
|
||||
|
||||
Run this script to test-render all the translation files that have been changed (when compared to the `main` branch).
|
||||
|
||||
---
|
||||
|
||||
|
||||
### [`kill-server-for-jest.js`](kill-server-for-jest.js)
|
||||
|
||||
|
||||
|
|
@ -577,13 +508,6 @@ This script is intended to be used as a git "prepush" hook. If the current branc
|
|||
---
|
||||
|
||||
|
||||
### [`prevent-translation-commits.js`](prevent-translation-commits.js)
|
||||
|
||||
This script is run as a git precommit hook (installed by husky after npm install). It detects changes to files the in the translations folder and prevents the commit if any changes exist.
|
||||
|
||||
---
|
||||
|
||||
|
||||
### [`purge-fastly`](purge-fastly)
|
||||
|
||||
Run this script to manually purge the Fastly cache. Note this script requires a `FASTLY_SERVICE_ID` and `FASTLY_TOKEN` in your `.env` file.
|
||||
|
|
|
|||
|
|
@ -2,6 +2,4 @@
|
|||
|
||||
This directory stores scripts that modify content and/or data files. Because
|
||||
writers are updating content all the time, scripts in here require more
|
||||
cross-team coordination and planning before they are run. Make sure to consider
|
||||
whether a script added here also needs to be run on translation files or if we
|
||||
can wait for the changes to come in through out translation automation.
|
||||
cross-team coordination and planning before they are run. Make sure to consider if we can wait for the changes to come in through out translation automation.
|
||||
|
|
|
|||
|
|
@ -1,106 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
// [start-readme]
|
||||
//
|
||||
// Run this script to fix known frontmatter errors by copying values from english file
|
||||
// Currently only fixing errors in: 'type', 'changelog'
|
||||
// Please double check the changes created by this script before committing.
|
||||
//
|
||||
// [end-readme]
|
||||
|
||||
import path from 'path'
|
||||
import { execSync } from 'child_process'
|
||||
import { get, set } from 'lodash-es'
|
||||
import fs from 'fs'
|
||||
import fm from '../../lib/frontmatter.js'
|
||||
import matter from 'gray-matter'
|
||||
import chalk from 'chalk'
|
||||
import yaml from 'js-yaml'
|
||||
import releaseNotesSchema from '../../tests/helpers/schemas/release-notes-schema.js'
|
||||
import revalidator from 'revalidator'
|
||||
|
||||
main()
|
||||
|
||||
async function main() {
|
||||
const fixableFmProps = Object.keys(fm.schema.properties)
|
||||
.filter((property) => !fm.schema.properties[property].translatable)
|
||||
.sort()
|
||||
const fixableYmlProps = ['date']
|
||||
|
||||
const loadAndValidateContent = async (path, schema) => {
|
||||
let fileContents
|
||||
try {
|
||||
fileContents = await fs.promises.readFile(path, 'utf8')
|
||||
} catch (e) {
|
||||
if (fs.existsSync(path)) {
|
||||
console.error(e.message)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
if (path.endsWith('yml')) {
|
||||
let data
|
||||
let errors = []
|
||||
try {
|
||||
data = yaml.load(fileContents)
|
||||
} catch {}
|
||||
if (data && schema) {
|
||||
;({ errors } = revalidator.validate(data, schema))
|
||||
}
|
||||
return { data, errors, content: null }
|
||||
} else {
|
||||
return fm(fileContents)
|
||||
}
|
||||
}
|
||||
|
||||
const cmd =
|
||||
'git -c diff.renameLimit=10000 diff --name-only origin/main | egrep "^translations/.*/(content/.+.md|data/release-notes/.*.yml)$"'
|
||||
|
||||
const maxBuffer = 1024 * 1024 * 2 // twice the default value
|
||||
const changedFilesRelPaths = execSync(cmd, { maxBuffer }).toString().split('\n')
|
||||
|
||||
for (const relPath of changedFilesRelPaths) {
|
||||
// Skip READMEs
|
||||
if (!relPath || relPath.endsWith('README.md')) continue
|
||||
|
||||
// find the corresponding english file by removing the first 2 path segments: /translation/<language code>
|
||||
const engAbsPath = relPath.split(path.sep).slice(2).join(path.sep)
|
||||
|
||||
const localisedResult = await loadAndValidateContent(relPath, releaseNotesSchema)
|
||||
if (!localisedResult) continue
|
||||
const { data, errors, content } = localisedResult
|
||||
|
||||
const fixableProps = relPath.endsWith('yml') ? fixableYmlProps : fixableFmProps
|
||||
|
||||
const fixableErrors = errors.filter(({ property }) => {
|
||||
const prop = property.split('.')
|
||||
return fixableProps.includes(prop[0])
|
||||
})
|
||||
|
||||
if (!data || fixableErrors.length === 0) continue
|
||||
|
||||
const engResult = await loadAndValidateContent(engAbsPath)
|
||||
if (!engResult) continue
|
||||
const { data: engData } = engResult
|
||||
|
||||
console.log(chalk.bold(relPath))
|
||||
|
||||
const newData = data
|
||||
|
||||
fixableErrors.forEach(({ property, message }) => {
|
||||
const correctValue = get(engData, property)
|
||||
console.log(chalk.red(` error message: [${property}] ${message}`))
|
||||
console.log(` fix property [${property}]: ${get(data, property)} -> ${correctValue}`)
|
||||
set(newData, property, correctValue)
|
||||
})
|
||||
|
||||
let toWrite
|
||||
if (content) {
|
||||
toWrite = matter.stringify(content, newData, { lineWidth: 10000, forceQuotes: true })
|
||||
} else {
|
||||
toWrite = yaml.dump(newData, { lineWidth: 10000, forceQuotes: true })
|
||||
}
|
||||
|
||||
fs.writeFileSync(relPath, toWrite)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
// [start-readme]
|
||||
//
|
||||
// Run this script to fix known frontmatter errors by copying values from english file
|
||||
// Translatable properties are designated in the frontmatter JSON schema
|
||||
//
|
||||
// [end-readme]
|
||||
|
||||
import path from 'path'
|
||||
import fs from 'fs/promises'
|
||||
import matter from 'gray-matter'
|
||||
import walk from 'walk-sync'
|
||||
import fm from '../../lib/frontmatter.js'
|
||||
|
||||
// Run!
|
||||
main()
|
||||
|
||||
async function main() {
|
||||
const translatedMarkdownFiles = walk('translations')
|
||||
.filter((filename) => {
|
||||
return (
|
||||
filename.includes('/content/') &&
|
||||
filename.endsWith('.md') &&
|
||||
!filename.endsWith('README.md')
|
||||
)
|
||||
})
|
||||
.map((filename) => `translations/${filename}`)
|
||||
|
||||
console.log(
|
||||
(
|
||||
await Promise.all(
|
||||
translatedMarkdownFiles.map(async (relPath) =>
|
||||
updateTranslatedMarkdownFile(relPath).catch((e) => `Error in ${relPath}: ${e.message}`)
|
||||
)
|
||||
)
|
||||
)
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
)
|
||||
}
|
||||
|
||||
async function extractFrontmatter(path) {
|
||||
const fileContents = await fs.readFile(path, 'utf8')
|
||||
return fm(fileContents)
|
||||
}
|
||||
|
||||
async function updateTranslatedMarkdownFile(relPath) {
|
||||
// find the corresponding english file by removing the first 2 path segments: /translations/<language code>
|
||||
const engAbsPath = relPath.split(path.sep).slice(2).join(path.sep)
|
||||
|
||||
// Load frontmatter from the source english file
|
||||
let englishFrontmatter
|
||||
try {
|
||||
englishFrontmatter = await extractFrontmatter(engAbsPath)
|
||||
} catch {
|
||||
// This happens when an English file has been moved or deleted and translations are not in sync.
|
||||
// It does mean this script will not homogenous those translated files, but the docs site does not
|
||||
// load translated files that don't correlate to an English file, so those translated files can't break things.
|
||||
// return `${relPath}: English file does not exist: ${engAbsPath}`
|
||||
return // silence
|
||||
}
|
||||
|
||||
const localisedFrontmatter = await extractFrontmatter(relPath)
|
||||
if (!localisedFrontmatter) return `${relPath}: No localised frontmatter`
|
||||
|
||||
// Look for differences between the english and localised non-translatable properties
|
||||
let overwroteSomething = false
|
||||
for (const prop in localisedFrontmatter.data) {
|
||||
if (
|
||||
!fm.schema.properties[prop].translatable &&
|
||||
englishFrontmatter.data[prop] &&
|
||||
localisedFrontmatter.data[prop] !== englishFrontmatter.data[prop]
|
||||
) {
|
||||
localisedFrontmatter.data[prop] = englishFrontmatter.data[prop]
|
||||
overwroteSomething = true
|
||||
}
|
||||
}
|
||||
|
||||
// rewrite the localised file, if it changed
|
||||
if (overwroteSomething) {
|
||||
const toWrite = matter.stringify(localisedFrontmatter.content, localisedFrontmatter.data, {
|
||||
lineWidth: 10000,
|
||||
forceQuotes: true,
|
||||
})
|
||||
await fs.writeFile(relPath, toWrite)
|
||||
|
||||
// return `${relPath}: updated`
|
||||
// silence
|
||||
}
|
||||
}
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import { program } from 'commander'
|
||||
import fs from 'fs'
|
||||
import languages from '../../lib/languages.js'
|
||||
|
||||
const defaultWorkflowUrl = [
|
||||
process.env.GITHUB_SERVER_URL,
|
||||
process.env.GITHUB_REPOSITORY,
|
||||
'actions/runs',
|
||||
process.env.GITHUB_RUN_ID,
|
||||
].join('/')
|
||||
|
||||
const reportTypes = {
|
||||
'pull-request-body': pullRequestBodyReport,
|
||||
csv: csvReport,
|
||||
}
|
||||
|
||||
program
|
||||
.description('Reads a translation batch log and generates a report')
|
||||
.requiredOption('--language <language>', 'The language to compare')
|
||||
.requiredOption('--log-file <log-file>', 'The batch log file')
|
||||
.requiredOption(
|
||||
'--report-type <report-type>',
|
||||
'The batch log file, I.E: ' + Object.keys(reportTypes).join(', ')
|
||||
)
|
||||
.option('--workflow-url <workflow-url>', 'The workflow url', defaultWorkflowUrl)
|
||||
.option('--csv-path <file-path>', 'The path to the CSV file')
|
||||
.parse(process.argv)
|
||||
|
||||
const options = program.opts()
|
||||
const language = languages[options.language]
|
||||
const { logFile, workflowUrl, reportType, csvPath } = options
|
||||
|
||||
if (!Object.keys(reportTypes).includes(reportType)) {
|
||||
throw new Error(`Invalid report type: ${reportType}`)
|
||||
}
|
||||
|
||||
const logFileContents = fs.readFileSync(logFile, 'utf8')
|
||||
|
||||
const revertLines = logFileContents
|
||||
.split('\n')
|
||||
.filter((line) => line.match(/^(-> reverted to English)|^(-> removed)/))
|
||||
.filter((line) => line.match(language.dir))
|
||||
|
||||
const reportEntries = revertLines.sort().map((line) => {
|
||||
const [, file, reason] = line.match(/^-> (?:reverted to English|removed): (.*) Reason: (.*)$/)
|
||||
return { file, reason }
|
||||
})
|
||||
|
||||
function pullRequestBodyReport() {
|
||||
return [
|
||||
`New translation batch for ${language.name}. Product of [this workflow](${workflowUrl}).
|
||||
|
||||
## ${reportEntries.length} files reverted.
|
||||
|
||||
You can see the log in [\`${csvPath}\`](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/blob/${csvPath}).`,
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
function csvReport() {
|
||||
const lines = reportEntries.map(({ file, reason }) => {
|
||||
return [file, reason].join(',')
|
||||
})
|
||||
|
||||
return ['file,reason', lines].flat().join('\n')
|
||||
}
|
||||
|
||||
console.log(reportTypes[reportType]())
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import { program } from 'commander'
|
||||
import { execFileSync } from 'child_process'
|
||||
import { languageFiles, compareLiquidTags } from './msft-tokens.js'
|
||||
import languages from '../../lib/languages.js'
|
||||
|
||||
program
|
||||
.description('show-liquid-tags-diff')
|
||||
.requiredOption('-l, --language <language>', 'The language to compare')
|
||||
.option('-d, --dry-run', 'Just pretend to reset files')
|
||||
.parse(process.argv)
|
||||
|
||||
function resetFiles(files) {
|
||||
console.log(`Reseting ${files.length} files:`)
|
||||
|
||||
const dryRun = program.opts().dryRun ? '--dry-run' : ''
|
||||
|
||||
files.forEach((file) => {
|
||||
const cmd = 'script/i18n/reset-translated-file.js'
|
||||
const args = [file, '--reason', 'broken liquid tags', dryRun]
|
||||
execFileSync(cmd, args, { stdio: 'inherit' })
|
||||
})
|
||||
}
|
||||
|
||||
function deleteFiles(files) {
|
||||
console.log(`Deleting ${files.length} files:`)
|
||||
|
||||
const dryRun = program.opts().dryRun ? '--dry-run' : ''
|
||||
|
||||
files.forEach((file) => {
|
||||
const cmd = 'script/i18n/reset-translated-file.js'
|
||||
const args = [
|
||||
file,
|
||||
'--remove',
|
||||
'--reason',
|
||||
'file deleted because it no longer exists in main',
|
||||
dryRun,
|
||||
]
|
||||
execFileSync(cmd, args, { stdio: 'inherit' })
|
||||
})
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = program.opts()
|
||||
const language = languages[options.language]
|
||||
|
||||
if (!language) {
|
||||
throw new Error(`Language ${options.language} not found`)
|
||||
}
|
||||
|
||||
// languageFiles() returns an array indexed as follows:
|
||||
// [0]: intersection of the files that exist in both main and the language-specific branch
|
||||
// [1]: files that exist only in the language-specific branch, not in main
|
||||
const allContentFiles = languageFiles(language, 'content')
|
||||
const allDataFiles = languageFiles(language, 'data')
|
||||
const files = [allContentFiles[0], allDataFiles[0]].flat()
|
||||
const nonexitentFiles = [allContentFiles[1], allDataFiles[1]].flat()
|
||||
const brokenFiles = []
|
||||
|
||||
files.forEach((file) => {
|
||||
try {
|
||||
// it throws error if the the syntax is invalid
|
||||
const comparison = compareLiquidTags(file, language)
|
||||
|
||||
if (comparison.diff.count === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
brokenFiles.push(comparison.translation)
|
||||
} catch (e) {
|
||||
brokenFiles.push(e.filePath)
|
||||
}
|
||||
})
|
||||
|
||||
await resetFiles(brokenFiles)
|
||||
await deleteFiles(nonexitentFiles)
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
import walk from 'walk-sync'
|
||||
import { Tokenizer } from 'liquidjs'
|
||||
import { readFileSync } from 'fs'
|
||||
import gitDiff from 'git-diff'
|
||||
import _ from 'lodash'
|
||||
|
||||
function getGitDiff(a, b) {
|
||||
return gitDiff(a, b, { flags: '--text --ignore-all-space' })
|
||||
}
|
||||
|
||||
function getMissingLines(diff) {
|
||||
return diff
|
||||
.split('\n')
|
||||
.filter((line) => line.startsWith('-'))
|
||||
.map((line) => line.replace('-', ''))
|
||||
}
|
||||
|
||||
function getExceedingLines(diff) {
|
||||
return diff
|
||||
.split('\n')
|
||||
.filter((line) => line.startsWith('+'))
|
||||
.map((line) => line.replace('+', ''))
|
||||
}
|
||||
|
||||
export function languageFiles(language, folder = 'content') {
|
||||
const englishFiles = walk(folder, { directories: false })
|
||||
const languageFiles = walk(`${language.dir}/${folder}`, { directories: false })
|
||||
return [
|
||||
_.intersection(englishFiles, languageFiles).map((file) => `${folder}/${file}`),
|
||||
_.difference(languageFiles, englishFiles).map((file) => `${language.dir}/${folder}/${file}`), // returns languageFiles not included in englishFiles
|
||||
]
|
||||
}
|
||||
|
||||
export function compareLiquidTags(file, language) {
|
||||
const translation = `${language.dir}/${file}`
|
||||
const sourceTokens = getTokensFromFile(file).rejectType('html')
|
||||
const otherFileTokens = getTokensFromFile(translation).rejectType('html')
|
||||
const diff = sourceTokens.diff(otherFileTokens)
|
||||
|
||||
return {
|
||||
file,
|
||||
translation,
|
||||
diff,
|
||||
}
|
||||
}
|
||||
|
||||
function getTokens(contents) {
|
||||
const tokenizer = new Tokenizer(contents)
|
||||
return new Tokens(...tokenizer.readTopLevelTokens())
|
||||
}
|
||||
|
||||
export function getTokensFromFile(filePath) {
|
||||
const contents = readFileSync(filePath, 'utf8')
|
||||
try {
|
||||
return new Tokens(...getTokens(contents))
|
||||
} catch (e) {
|
||||
const error = new Error(`Error parsing ${filePath}: ${e.message}`)
|
||||
error.filePath = filePath
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export class Tokens extends Array {
|
||||
rejectType(tagType) {
|
||||
return this.filter(
|
||||
(token) => token.constructor.name.toUpperCase() !== `${tagType}Token`.toUpperCase()
|
||||
)
|
||||
}
|
||||
|
||||
onlyText() {
|
||||
return this.map((token) => token.getText())
|
||||
}
|
||||
|
||||
diff(otherTokens) {
|
||||
const a = this.onlyText().sort()
|
||||
const b = otherTokens.onlyText().sort()
|
||||
|
||||
const diff = getGitDiff(a.join('\n'), b.join('\n'))
|
||||
|
||||
if (!diff) {
|
||||
return { count: 0, missing: [], exceeding: [], output: '' }
|
||||
}
|
||||
|
||||
const missing = getMissingLines(diff)
|
||||
const exceeding = getExceedingLines(diff)
|
||||
const count = exceeding.length + missing.length
|
||||
|
||||
return { count, missing, exceeding, output: diff }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import walk from 'walk-sync'
|
||||
import { program } from 'commander'
|
||||
import languages from '../../lib/languages.js'
|
||||
|
||||
program
|
||||
.description(
|
||||
`Removes any file in the translations directory that doesn't have a 1-1 mapping with an English file in the content directory`
|
||||
)
|
||||
.option('-d, --dry-run', `List the files that will be deleted, but don't remove them).`)
|
||||
.parse(process.argv)
|
||||
|
||||
const languageDir = Object.keys(languages)
|
||||
.filter((language) => !languages[language].wip && language !== 'en')
|
||||
.map((language) => languages[language].dir)
|
||||
|
||||
main()
|
||||
|
||||
async function main() {
|
||||
const listOfContentFiles = walk(path.join(process.cwd(), 'content'), {
|
||||
includeBasePath: false,
|
||||
directories: false,
|
||||
})
|
||||
|
||||
const translatedFilePaths = []
|
||||
languageDir.forEach((directory) => {
|
||||
const listOfFiles = walk(path.join(directory, 'content'), {
|
||||
includeBasePath: true,
|
||||
directories: false,
|
||||
}).map((path) => path.replace(process.cwd(), ''))
|
||||
translatedFilePaths.push(...listOfFiles)
|
||||
})
|
||||
|
||||
let outOfSyncFilesCount = 0
|
||||
translatedFilePaths.forEach((translatedFilePath) => {
|
||||
const translationRelativePath = translatedFilePath.split('/content/')[1]
|
||||
|
||||
// If there is a 1:1 mapping of translated file to english file
|
||||
// we're in sync, don't log
|
||||
if (listOfContentFiles.includes(translationRelativePath)) {
|
||||
return
|
||||
}
|
||||
|
||||
outOfSyncFilesCount++
|
||||
if (!program.opts().dryRun) {
|
||||
fs.unlinkSync(translatedFilePath)
|
||||
} else {
|
||||
console.log(translatedFilePath)
|
||||
}
|
||||
})
|
||||
|
||||
console.log(`Out of sync file size: ${outOfSyncFilesCount}`)
|
||||
}
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
// [start-readme]
|
||||
//
|
||||
// This is a convenience script for replacing the contents of translated
|
||||
// files with the English content from their corresponding source file.
|
||||
//
|
||||
// Usage:
|
||||
// script/i18n/reset-translated-file.js <filename>
|
||||
//
|
||||
// Examples:
|
||||
//
|
||||
// $ script/i18n/reset-translated-file.js translations/es-XL/content/actions/index.md
|
||||
//
|
||||
// [end-readme]
|
||||
|
||||
import { program } from 'commander'
|
||||
import { execSync } from 'child_process'
|
||||
import assert from 'assert'
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import chalk from 'chalk'
|
||||
|
||||
program
|
||||
.description('reset translated files')
|
||||
.option(
|
||||
'-m, --prefer-main',
|
||||
'Reset file to the translated file, try using the file from `main` branch first, if not found (usually due to renaming), fall back to English source.'
|
||||
)
|
||||
.option('-rm, --remove', 'Remove the translated files altogether')
|
||||
.option('-d, --dry-run', 'Just pretend to reset files')
|
||||
.option('-r, --reason <reason>', 'A reason why the file is getting reset')
|
||||
.parse(process.argv)
|
||||
|
||||
const dryRun = program.opts().dryRun
|
||||
const reason = program.opts().reason
|
||||
const reasonMessage = reason ? `Reason: ${reason}` : ''
|
||||
|
||||
const resetToEnglishSource = (translationFilePath) => {
|
||||
assert(
|
||||
translationFilePath.startsWith('translations/'),
|
||||
'path argument must be in the format `translations/<lang>/path/to/file`'
|
||||
)
|
||||
|
||||
if (program.opts().remove) {
|
||||
if (!dryRun) {
|
||||
const fullPath = path.join(process.cwd(), translationFilePath)
|
||||
fs.unlinkSync(fullPath)
|
||||
}
|
||||
console.log('-> removed: %s %s', translationFilePath, reasonMessage)
|
||||
return
|
||||
}
|
||||
if (!fs.existsSync(translationFilePath)) {
|
||||
return
|
||||
}
|
||||
|
||||
const relativePath = translationFilePath.split(path.sep).slice(2).join(path.sep)
|
||||
const englishFile = path.join(process.cwd(), relativePath)
|
||||
|
||||
if (!dryRun && !fs.existsSync(englishFile)) {
|
||||
fs.unlinkSync(translationFilePath)
|
||||
return
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
// it is important to replace the file with English source instead of
|
||||
// removing it, and relying on the fallback, because redired_from frontmatter
|
||||
// won't work in fallbacks
|
||||
const englishContent = fs.readFileSync(englishFile, 'utf8')
|
||||
fs.writeFileSync(translationFilePath, englishContent)
|
||||
}
|
||||
|
||||
console.log(
|
||||
'-> reverted to English: %s %s',
|
||||
path.relative(process.cwd(), translationFilePath),
|
||||
reasonMessage
|
||||
)
|
||||
}
|
||||
|
||||
const [pathArg] = program.args
|
||||
assert(pathArg, 'first arg must be a target filename')
|
||||
|
||||
// Is the arg a fully-qualified path?
|
||||
const relativePath = fs.existsSync(pathArg) ? path.relative(process.cwd(), pathArg) : pathArg
|
||||
|
||||
if (program.opts().preferMain) {
|
||||
try {
|
||||
if (!dryRun) {
|
||||
execSync(`git checkout main -- ${relativePath}`, { stdio: 'pipe' })
|
||||
}
|
||||
console.log('-> reverted to file from main branch: %s %s', relativePath, reasonMessage)
|
||||
} catch (e) {
|
||||
if (e.message.includes('pathspec')) {
|
||||
console.warn(
|
||||
chalk.red(
|
||||
`cannot find ${relativePath} in main branch (likely because it was renamed); falling back to English source file.`
|
||||
)
|
||||
)
|
||||
resetToEnglishSource(relativePath)
|
||||
} else {
|
||||
console.warn(e.message)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
resetToEnglishSource(relativePath)
|
||||
}
|
||||
|
|
@ -1,124 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
// [start-readme]
|
||||
//
|
||||
// Run this script to test-render all the translation files that have been changed (when compared to the `main` branch).
|
||||
//
|
||||
// [end-readme]
|
||||
|
||||
import renderContent from '../../lib/render-content/index.js'
|
||||
import loadSiteData from '../../lib/site-data.js'
|
||||
import { loadPages } from '../../lib/page-data.js'
|
||||
import languages from '../../lib/languages.js'
|
||||
import { promisify } from 'util'
|
||||
import ChildProcess, { execSync } from 'child_process'
|
||||
import fs from 'fs'
|
||||
import frontmatter from '../../lib/frontmatter.js'
|
||||
import chalk from 'chalk'
|
||||
import { YAMLException } from 'js-yaml'
|
||||
|
||||
const fmSchemaProperties = frontmatter.schema.properties
|
||||
const exec = promisify(ChildProcess.exec)
|
||||
|
||||
main()
|
||||
|
||||
async function main() {
|
||||
const siteData = await loadAndPatchSiteData()
|
||||
const pages = await loadPages()
|
||||
const contextByLanguage = {}
|
||||
for (const lang in languages) {
|
||||
const langObj = languages[lang]
|
||||
const [langCode] = langObj.dir === '' ? 'en' : langObj.dir.split('/').slice(1)
|
||||
if (!langCode) continue
|
||||
contextByLanguage[langCode] = {
|
||||
site: siteData[langObj.code].site,
|
||||
currentLanguage: langObj.code,
|
||||
currentVersion: 'free-pro-team@latest',
|
||||
}
|
||||
}
|
||||
|
||||
const changedFilesRelPaths = execSync(
|
||||
'git -c diff.renameLimit=10000 diff --name-only origin/main | egrep "^translations/.*/.+(.md|.yml)$"',
|
||||
{ maxBuffer: 1024 * 1024 * 100 }
|
||||
)
|
||||
.toString()
|
||||
.split('\n')
|
||||
.filter((path) => path !== '' && !path.endsWith('README.md'))
|
||||
.sort()
|
||||
|
||||
console.log(`Found ${changedFilesRelPaths.length} translated files.`)
|
||||
|
||||
for (const relPath of changedFilesRelPaths) {
|
||||
const lang = relPath.split('/')[1]
|
||||
const context = {
|
||||
...contextByLanguage[lang],
|
||||
pages,
|
||||
page: pages.find((page) => {
|
||||
const pageRelPath = `${languages[page.languageCode].dir}/content/${page.relativePath}`
|
||||
return pageRelPath === relPath
|
||||
}),
|
||||
redirects: {},
|
||||
}
|
||||
|
||||
// specifically test rendering data/variables files for broken liquid
|
||||
if (relPath.includes('data/variables')) {
|
||||
const fileContents = await fs.promises.readFile(relPath, 'utf8')
|
||||
const { content } = frontmatter(fileContents)
|
||||
|
||||
try {
|
||||
await renderContent.liquid.parseAndRender(content, context)
|
||||
} catch (err) {
|
||||
console.log(chalk.bold(relPath))
|
||||
console.log(chalk.red(` error message: ${err.message}`))
|
||||
}
|
||||
}
|
||||
|
||||
if (!context.page && !relPath.includes('data/reusables')) continue
|
||||
const fileContents = await fs.promises.readFile(relPath, 'utf8')
|
||||
const { data, content } = frontmatter(fileContents)
|
||||
const translatableFm = Object.keys(data).filter((key) => fmSchemaProperties[key].translatable)
|
||||
try {
|
||||
// test the content
|
||||
await renderContent.liquid.parseAndRender(content, context)
|
||||
// test each translatable frontmatter property
|
||||
for (const key of translatableFm) {
|
||||
await renderContent.liquid.parseAndRender(data[key], context)
|
||||
}
|
||||
} catch (err) {
|
||||
console.log(chalk.bold(relPath))
|
||||
console.log(chalk.red(` error message: ${err.message}`))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function loadAndPatchSiteData(filesWithKnownIssues = {}) {
|
||||
try {
|
||||
const siteData = loadSiteData()
|
||||
return siteData
|
||||
} catch (error) {
|
||||
if (error instanceof YAMLException && error.mark) {
|
||||
const relPath = error.mark.name
|
||||
if (!filesWithKnownIssues[relPath]) {
|
||||
// Note the file as problematic
|
||||
filesWithKnownIssues[relPath] = true
|
||||
|
||||
// This log is important as it will get ${relPath} written to a logfile
|
||||
console.log(chalk.bold(relPath))
|
||||
console.log(chalk.red(` error message: ${error.toString()}`))
|
||||
|
||||
// Reset the file
|
||||
console.warn(`resetting file "${relPath}" due to loadSiteData error: ${error.toString()}`)
|
||||
await exec(
|
||||
`script/i18n/reset-translated-file.js --prefer-main ${relPath} --reason="loadSiteData error"`
|
||||
)
|
||||
|
||||
// Try to load the site data again
|
||||
return loadAndPatchSiteData(filesWithKnownIssues)
|
||||
} else {
|
||||
console.error(`FATAL: Tried to reset file "${relPath}" but still had errors`)
|
||||
}
|
||||
}
|
||||
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
|
@ -8,4 +8,6 @@
|
|||
|
||||
source script/check-for-node
|
||||
|
||||
# TODO would need git clones from the language repos
|
||||
|
||||
npm run start-all-languages
|
||||
|
|
|
|||
|
|
@ -1,82 +0,0 @@
|
|||
import { expect } from '@jest/globals'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
import { getTokensFromFile, Tokens } from '../../../script/i18n/msft-tokens'
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
function getFixturePath(name) {
|
||||
return path.join(__dirname, '../..', 'fixtures', name)
|
||||
}
|
||||
|
||||
describe('getTokensFromFile', () => {
|
||||
let fixturePath
|
||||
let tokens
|
||||
|
||||
beforeEach(() => {
|
||||
fixturePath = getFixturePath('liquid-tags/minimal-conditional.md')
|
||||
tokens = getTokensFromFile(fixturePath)
|
||||
})
|
||||
|
||||
describe('getTokensFromFile', () => {
|
||||
it('returns all the tokens from a template file', () => {
|
||||
expect(tokens.length).toEqual(7)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Tokens', () => {
|
||||
describe('.rejectType', () => {
|
||||
it('rejects tokens of a particular type', () => {
|
||||
const nonHtmlTokens = tokens.rejectType('html')
|
||||
|
||||
expect(nonHtmlTokens.length).toEqual(3)
|
||||
})
|
||||
})
|
||||
|
||||
describe('.diff', () => {
|
||||
let tokens
|
||||
let otherTokens
|
||||
let reverseTokens
|
||||
|
||||
const addTokens = (collection, elements) => {
|
||||
elements.forEach((element) => {
|
||||
collection.push({ getText: () => element })
|
||||
})
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
tokens = new Tokens()
|
||||
otherTokens = new Tokens()
|
||||
reverseTokens = new Tokens()
|
||||
addTokens(tokens, ['apples', 'bananas', 'oranges'])
|
||||
addTokens(otherTokens, ['apples', 'oranges'])
|
||||
addTokens(reverseTokens, ['oranges', 'bananas', 'apples'])
|
||||
})
|
||||
|
||||
it('shows elements that are missing', () => {
|
||||
const diff = tokens.diff(otherTokens)
|
||||
|
||||
expect(diff.count).toEqual(1)
|
||||
expect(diff.missing).toEqual(['bananas'])
|
||||
})
|
||||
|
||||
it('shows elements that are exceeding', () => {
|
||||
const diff = otherTokens.diff(tokens)
|
||||
|
||||
expect(diff.count).toEqual(1)
|
||||
expect(diff.exceeding).toEqual(['bananas'])
|
||||
})
|
||||
|
||||
it('shows no difference when collections are the same', () => {
|
||||
const diff = tokens.diff(tokens)
|
||||
|
||||
expect(diff.count).toEqual(0)
|
||||
})
|
||||
|
||||
it('shows no difference when tokens are in different order', () => {
|
||||
const diff = tokens.diff(reverseTokens)
|
||||
expect(diff.count).toEqual(0)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
Загрузка…
Ссылка в новой задаче