feat: Crowdin integration

Integrates with Crowdin to enable i18n of the site. This PR changes the
source of truth for Crowdin and moves it to this repository instead of
relying on `electron/i18n`.

Additionally it uses Crowdin's CLI to do the upload/download of assets
making it more reliable.

The website is built on locale at a time via `yarn i18n:build`.
Otherwise the process crashed with an out of memory error. The regular
`yarn build` command still compiles the `en` locale.

Because we cannot get notifications when there are new translations
avaiable, there is a GitHub workflow (`update-i18n-deploy.yml`) that
downloads the content every few minutes, builds, and deploy. To speed up
this process, the previous generated assets are download. In local tests
this reduces the build times from 250s to 40s so the whole process
should take about 5 minutes.

The previous generated content is stored in Azure Storage. Because this
is a static website it makes more sense than having a dyno and will make
it easier to:
- deploy multiple locales at the same time if we still need to speed up
  the process
- have versioned docs because we just need to "take a snapshot" and
  publish to a different folder

The current live site is still not using this storage but will soon-ish.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

Fix #64

.github/workflows/en-deploy.yml

@@ -0,0 +1,39 @@
+name: Deploy EN
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  tests:
+    name: Test and Deploy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Use Node.js 14
+        uses: actions/setup-node@v2
+        with:
+          node-version: 14
+
+      - name: Install dependencies
+        uses: bahmutov/npm-install@HEAD
+
+      - name: Test
+        run: yarn test
+        env:
+          CI: true
+
+      - name: Download cache
+        run: ./scripts/bin/azcopy copy "https://electronjsorg.blob.core.windows.net/%24web/*?${{ SSA }}" "./build" --recursive
+        env:
+          SSA: ${{ secrets.SSA }}
+
+      - name: Build EN
+        run: yarn i18n:build en
+
+      - name: Deploy
+        run: ./scripts/bin/azcopy copy "./build/*" "https://electronjsorg.blob.core.windows.net/%24web?${{ SSA }}" --recursive
+        env:
+          SAS: ${{ secrets.SSA }}

.github/workflows/test.yml

@@ -4,9 +4,6 @@ on:
   pull_request:
     branches:
       - main
-  push:
-    branches:
-      - main
 
 jobs:
   tests:

.github/workflows/update-i18n-deploy.yml

@@ -0,0 +1,38 @@
+name: 'Update i18n deploy'
+
+on:
+  schedule:
+    - cron: '*/15 * * * *'
+
+jobs:
+  deploy:
+    name: 'Build and deploy localized site'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Use Node.js 14
+        uses: actions/setup-node@v2
+        with:
+          node-version: 14
+
+      - name: Install dependencies
+        uses: bahmutov/npm-install@HEAD
+
+      - name: Download crowdin translation
+        run: yarn i18n:download
+        env:
+          CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}
+
+      - name: Download cache
+        run: ./scripts/bin/azcopy copy "https://electronjsorg.blob.core.windows.net/%24web/*?${{ SSA }}" "./build" --recursive
+        env:
+          SSA: ${{ secrets.SSA }}
+
+      - name: Build
+        run: yarn i18n:build
+
+      - name: Deploy
+        run: ./scripts/bin/azcopy copy "./build/*" "https://electronjsorg.blob.core.windows.net/%24web?${{ SSA }}" --recursive
+        env:
+          SAS: ${{ secrets.SSA }}

.gitignore

@@ -4,4 +4,6 @@ node_modules
 .env
 .vscode/settings.json
 build/
-content/
+content/
+i18n/
+!i18n/en/

crowdin.yml

@@ -0,0 +1,21 @@
+project_id: '273870'
+api_token_env: 'CROWDIN_PERSONAL_TOKEN'
+preserve_hierarchy: true
+files: [
+    # JSON translation files
+    {
+      source: '/i18n/en/**/*',
+      translation: '/i18n/%two_letters_code%/**/%original_file_name%',
+    },
+    # Docs Markdown files
+    {
+      source: '/docs/**/*',
+      translation: '/i18n/%two_letters_code%/docusaurus-plugin-content-docs/current/**/%original_file_name%',
+      ignore: ['/docs/**/fiddles', '/docs/**/images'],
+    },
+    # Blog Markdown files
+    {
+      source: '/blog/**/*',
+      translation: '/i18n/%two_letters_code%/docusaurus-plugin-content-blog/**/%original_file_name%',
+    },
+  ]

docusaurus.config.js

@@ -13,6 +13,10 @@ module.exports = {
   favicon: 'assets/img/favicon.ico',
   organizationName: 'electron',
   projectName: 'electron',
+  i18n: {
+    defaultLocale: 'en',
+    locales: ['en', 'de', 'es', 'fr', 'ja', 'pt', 'ru', 'zh'],
+  },
   themeConfig: {
     announcementBar: {
       id: 'to_old_docs',
@@ -52,6 +56,10 @@ module.exports = {
           label: 'Releases',
           position: 'right',
         },
+        {
+          type: 'localeDropdown',
+          position: 'right',
+        },
         {
           href: 'https://github.com/electron/electron',
           label: 'GitHub',
@@ -143,13 +151,17 @@ module.exports = {
         docs: {
           sidebarPath: require.resolve('./sidebars.js'),
           routeBasePath: '/docs/',
-          editUrl: ({docPath}) => {
+          editUrl: ({ docPath }) => {
             // TODO: remove when `latest/` is no longer hardcoded
             const fixedPath = docPath.replace('latest/', '');
             // TODO: versioning?
-            return `https://github.com/electron/electron/edit/main/docs/${fixedPath}`
+            return `https://github.com/electron/electron/edit/main/docs/${fixedPath}`;
           },
-          remarkPlugins: [fiddleEmbedder, apiLabels, [npm2yarn, { sync: true }]],
+          remarkPlugins: [
+            fiddleEmbedder,
+            apiLabels,
+            [npm2yarn, { sync: true }],
+          ],
         },
         blog: {
           // See `node_modules/@docusaurus/plugin-content-blog/src/pluginOptionSchema.ts` for full undocumented options

i18n/en/code.json

@@ -206,5 +206,13 @@
   "theme.tags.tagsPageTitle": {
     "message": "Tags",
     "description": "The title of the tag list page"
+  },
+  "theme.blog.archive.title": {
+    "message": "Archive",
+    "description": "The page & hero title of the blog archive page"
+  },
+  "theme.blog.archive.description": {
+    "message": "Archive",
+    "description": "The page & hero description of the blog archive page"
   }
 }

package.json

@@ -4,9 +4,13 @@
   "private": true,
   "license": "Apache-2.0",
   "scripts": {
+    "crowdin": "crowdin",
+    "i18n:upload": "crowdin upload sources",
+    "i18n:download": "crowdin download && node scripts/prepare-i18n-content.js",
+    "i18n:build": "node scripts/i18n-build.js",
     "docusaurus": "docusaurus",
     "start": "docusaurus start",
-    "build": "docusaurus build",
+    "build": "yarn pre-build && docusaurus build --locale en",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
     "clear": "docusaurus clear",
@@ -16,7 +20,7 @@
     "update-l10n-sources": "node scripts/update-l10n-sources.js",
     "lint": "prettier -c ./scripts/**/*.js",
     "test": "yarn lint && jest",
-    "prebuild": "node ./scripts/pre-build.js",
+    "pre-build": "node ./scripts/pre-build.js",
     "process-docs-changes": "node ./scripts/process-docs-changes.js",
     "update-pinned-version": "node ./scripts/update-pinned-version.js",
     "prepare": "husky install"
@@ -50,6 +54,7 @@
   "devDependencies": {
     "@actions/core": "^1.2.7",
     "@actions/github": "^4.0.0",
+    "@crowdin/cli": "3",
     "@types/jest": "^26.0.23",
     "@types/unist": "^2.0.3",
     "del": "^6.0.0",

scripts/i18n-build.js

@@ -0,0 +1,61 @@
+//@ts-check
+const fs = require('fs').promises;
+const { join } = require('path');
+const { execute } = require('./utils/execute');
+const {
+  i18n: { locales, defaultLocale },
+} = require('../docusaurus.config');
+
+const updateConfig = async (locale) => {
+  const baseUrl = locale !== defaultLocale ? `/${locale}/` : '/';
+  // Translations might not be completely in sync and we need to keep publishing
+  const onBrokenLinks = locale !== defaultLocale ? `warn` : `throw`;
+  const configPath = join(__dirname, '../docusaurus.config.js');
+
+  let docusaurusConfig = await fs.readFile(configPath, 'utf-8');
+
+  docusaurusConfig = docusaurusConfig
+    .replace(/baseUrl: '.*?',/, `baseUrl: '${baseUrl}',`)
+    .replace(/onBrokenLinks: '.*?',/, `onBrokenLinks: '${onBrokenLinks}',`);
+
+  await fs.writeFile(configPath, docusaurusConfig, 'utf-8');
+};
+
+const processLocale = async (locale) => {
+  const start = Date.now();
+  const outdir = locale !== defaultLocale ? `--out-dir build/${locale}` : '';
+  await execute(`yarn docusaurus build --locale ${locale} ${outdir}`);
+  console.log(`Locale ${locale} finished in ${(Date.now() - start) / 1000}s`);
+};
+
+/**
+ *
+ * @param {string} [locale]
+ */
+const start = async (locale) => {
+  const start = Date.now();
+
+  const localesToBuild = locale ? [locale] : locales;
+
+  console.log('Building the following locales:');
+  console.log(localesToBuild);
+
+  for (const locale of localesToBuild) {
+    try {
+      await updateConfig(locale);
+      await processLocale(locale);
+    } catch (e) {
+      // We catch instead of just stopping the process because we want to restore docusaurus.config.js
+      console.error(e);
+      // TODO: It will be nice to do some clean up and point to the right file and line
+      console.error(`Locale ${locale} failed. Please check the logs above.`)
+    }
+  }
+
+  // Restore `docusaurus.config.js` to the default values
+  await updateConfig(defaultLocale);
+
+  console.log(`Process finished in ${(Date.now() - start) / 1000}s`);
+};
+
+start(process.argv[2]);

scripts/prepare-i18n-content.js

@@ -0,0 +1,47 @@
+//@ts-check
+
+/**
+ * Takes care of downloading the documentation from the
+ * right places, and transform it to make it ready to
+ * be used by docusaurus.
+ */
+const path = require('path');
+const fs = require('fs-extra');
+
+const { addFrontmatter } = require('./tasks/add-frontmatter');
+const { fixContent } = require('./tasks/md-fixers');
+
+const DOCS_FOLDER = path.join('docs', 'latest');
+const {
+  i18n: { locales: configuredLocales },
+} = require('../docusaurus.config');
+
+const start = async () => {
+  const locales = new Set(configuredLocales);
+  locales.delete('en');
+  for (const locale of locales) {
+    const localeDocs = path.join(
+      'i18n',
+      locale,
+      'docusaurus-plugin-content-docs',
+      'current'
+    );
+    const staticResources = ['fiddles', 'images'];
+
+    console.log(`Copying static assets to ${locale}`);
+    for (const staticResource of staticResources) {
+      await fs.copy(
+        path.join(DOCS_FOLDER, staticResource),
+        path.join(localeDocs, 'latest', staticResource)
+      );
+    }
+
+    console.log(`Fixing markdown (${locale})`);
+    await fixContent(localeDocs, 'latest');
+
+    console.log(`Adding automatic frontmatter (${locale})`);
+    await addFrontmatter(path.join(localeDocs, 'latest'));
+  }
+};
+
+start();

scripts/process-docs-changes.js

@@ -14,6 +14,7 @@ if (
   process.exit(1);
 }
 
+const { execute } = require('./utils/execute');
 const { createPR, getChanges, pushChanges } = require('./utils/git-commands');
 
 const HEAD = 'main';
@@ -64,6 +65,9 @@ const processDocsChanges = async () => {
     console.log('package.json is not modified, skipping');
     return;
   } else {
+    console.log(`Uploading changes to Crowdin`);
+    await execute(`yarn crowdin:upload`);
+
     const newFiles = newDocFiles(output);
     if (newFiles.length > 0) {
       console.log(`New documents available:

scripts/tasks/add-frontmatter.js

@@ -87,7 +87,7 @@ const descriptionFromContent = (content) => {
 
     // The content of structures is often only bullet lists and no general description
     if (trimmedLine.startsWith('#') || trimmedLine.startsWith('*')) {
-      if (subHeader) {
+      if (subHeader && description.length > 0) {
         return cleanUpMarkdown(description.trim());
       } else {
         subHeader = true;
@@ -122,7 +122,10 @@ const addFrontMatter = (content, filepath) => {
     ? titleMatches[1].trim()
     : titleFromPath(filepath).trim();
 
-  const description = descriptionFromContent(content);
+  // The description of the files under `api/structures` is not meaningful so we ignore it
+  const description = filepath.includes('structures')
+    ? ''
+    : descriptionFromContent(content);
   const defaultSlug = path.basename(filepath, '.md');
 
   let slug;

scripts/tasks/md-fixers.js

@@ -66,12 +66,33 @@ const fiddleTransformer = (line) => {
   if (matches) {
     return `\`\`\`fiddle docs/latest/${matches[1]}`;
   } else if (hasNewPath) {
-    return line.replace(fiddlePathFixRegex, '```fiddle docs/latest/');
+    return (
+      line
+        .replace(fiddlePathFixRegex, '```fiddle docs/latest/')
+        // we could have a double transformation if the path is already the good one
+        // this happens especially with the i18n content
+        .replace('latest/latest', 'latest')
+    );
   } else {
     return line;
   }
 };
 
+/**
+ * Crowdin translations put markdown content right
+ * after HTML comments and thus breaking Docusaurus
+ * parse engine. We need to add a new EOL after `-->`
+ * is found.
+ * @param {string} line
+ */
+const newLineOnHTMLComment = (line) => {
+  // The `startsWith('*')` part is to prevent messing the document `api/native-theme.md` 😓
+  if (line.includes('-->') && !line.endsWith('-->') && !line.startsWith('*')) {
+    return line.replace('-->', '-->\n');
+  }
+  return line;
+};
+
 /**
  * Applies any transformation that can be executed line by line on
  * the document to make sure it is ready to be consumed by
@@ -83,7 +104,11 @@ const fiddleTransformer = (line) => {
 const transform = (doc) => {
   const lines = doc.split('\n');
   const newDoc = [];
-  const transformers = [apiTransformer, fiddleTransformer];
+  const transformers = [
+    apiTransformer,
+    fiddleTransformer,
+    newLineOnHTMLComment,
+  ];
 
   for (const line of lines) {
     const newLine = transformers.reduce((newLine, transformer) => {

scripts/update-l10n-sources.js

@@ -26,14 +26,13 @@ ${files.join('\n')}`);
     return;
   }
 
-  await del('i18n/en-US');
-  await execute('yarn write-translations --locale en-US');
+  await execute('yarn write-translations --locale en');
 
   const localeModified = (await getChanges()) !== output;
 
   if (localeModified) {
     const pleaseCommit =
-      'Contents in "/i18n/en-US/" have been modified. Please add the changes to your commit';
+      'Contents in "/i18n/en/" have been modified. Please add the changes to your commit';
     console.error('\x1b[31m%s\x1b', pleaseCommit);
     process.exit(1);
   }

yarn.lock

@@ -1310,6 +1310,13 @@
     exec-sh "^0.3.2"
     minimist "^1.2.0"
 
+"@crowdin/cli@3":
+  version "3.7.0"
+  resolved "https://registry.yarnpkg.com/@crowdin/cli/-/cli-3.7.0.tgz#d35b69e90b6a737a9de017423ac34c02291cdebb"
+  integrity sha512-7eje7V6BGMeW23ywbrYdvpdIIxG5O1WP2wit4MVP9EtuZMOfr1M0l9BnObbkSYK86UiZuoJFHs1Q1KoCWg1rlA==
+  dependencies:
+    shelljs "^0.8.4"
+
 "@docsearch/css@3.0.0-alpha.39":
   version "3.0.0-alpha.39"
   resolved "https://registry.yarnpkg.com/@docsearch/css/-/css-3.0.0-alpha.39.tgz#1ebd390d93e06aad830492f5ffdc8e05d058813f"