Extract l10n extract/merge to CI runnable job/script (#21734)
This commit is contained in:
Kevin Meinhardt
GitHub
Родитель
2baa8afb47
Коммит
e978c20fdc
|
|
@ -493,7 +493,6 @@ commands:
|
|||
- run:
|
||||
name: Set environment variables
|
||||
command: |
|
||||
echo export PYTHONPATH=src >> $BASH_ENV
|
||||
echo export CPUCOUNT=2 >> $BASH_ENV
|
||||
echo export NPM_CONFIG_PREFIX=/deps/ >> $BASH_ENV
|
||||
echo export CC=\"`python -c 'import sysconfig; print(sysconfig.get_config_var("CC"))'`\" >> $BASH_ENV
|
||||
|
|
@ -575,6 +574,17 @@ jobs:
|
|||
- run: pyenv rehash
|
||||
- run: make lint-codestyle
|
||||
|
||||
localization:
|
||||
<<: *defaults
|
||||
steps:
|
||||
- setup_container:
|
||||
wait_services: false
|
||||
install_node_dependencies: true
|
||||
install_python_dev_dependencies: true
|
||||
- run : make extract_locales
|
||||
- run: git diff
|
||||
|
||||
|
||||
devhub:
|
||||
<<: *defaults-with-services
|
||||
steps:
|
||||
|
|
@ -666,6 +676,7 @@ workflows:
|
|||
- main
|
||||
- reviewers-and-zadmin
|
||||
- es-tests
|
||||
- localization
|
||||
- release-master:
|
||||
filters:
|
||||
branches:
|
||||
|
|
|
|||
|
|
@ -80,12 +80,17 @@ cleanup_python_build_dir:
|
|||
# Work arounds "Multiple .dist-info directories" issue.
|
||||
rm -rf /deps/build/*
|
||||
|
||||
.PHONY: install_python_olympia_module
|
||||
install_python_olympia_module:
|
||||
# pep 517 mode (the default) breaks editable install in our project. https://github.com/mozilla/addons-server/issues/16144
|
||||
$(PIP_COMMAND) install --no-use-pep517 -e .
|
||||
|
||||
.PHONY: install_python_codestyle_dependencies
|
||||
install_python_codestyle_dependencies:
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/codestyle.txt
|
||||
|
||||
.PHONY: install_python_test_dependencies
|
||||
install_python_test_dependencies:
|
||||
install_python_test_dependencies: install_python_olympia_module
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/system.txt
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/tests.txt
|
||||
|
|
@ -94,8 +99,6 @@ install_python_test_dependencies:
|
|||
install_python_dev_dependencies: install_python_test_dependencies install_python_codestyle_dependencies
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/docs.txt
|
||||
# pep 517 mode (the default) breaks editable install in our project. https://github.com/mozilla/addons-server/issues/16144
|
||||
$(PIP_COMMAND) install --no-use-pep517 -e .
|
||||
|
||||
.PHONY: install_node_dependencies
|
||||
install_node_dependencies: install_node_js copy_node_js
|
||||
|
|
@ -235,6 +238,10 @@ format: ## Autoformat our codebase.
|
|||
ruff check --fix-only .
|
||||
ruff format .
|
||||
|
||||
.PHONY: extract_locales
|
||||
extract_locales: ## extracts and merges translation strings
|
||||
./scripts/run_l10n_extraction.sh
|
||||
|
||||
.PHONY: help_submake
|
||||
help_submake:
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile-docker | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
|
||||
|
|
|
|||
|
|
@ -19,11 +19,6 @@ set -o pipefail
|
|||
# Treat unset variables as an error an exit immediately.
|
||||
set -u
|
||||
|
||||
# Extraction needs our django settings for jinja, so we need a django settings
|
||||
# module set. Since this command is meant to be run in local envs, we use
|
||||
# "settings".
|
||||
DJANGO_SETTINGS_MODULE=settings
|
||||
|
||||
INITIAL_GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||
GIT_CHANGES=$(git status --porcelain)
|
||||
GIT_REMOTE="https://github.com/mozilla/addons-server.git" # Upstream.
|
||||
|
|
@ -33,10 +28,11 @@ BRANCH_NAME=""
|
|||
ROBOT_EMAIL="addons-dev-automation+github@mozilla.com"
|
||||
ROBOT_NAME="Mozilla Add-ons Robot"
|
||||
|
||||
# gettext flags
|
||||
CLEAN_FLAGS="--no-obsolete --width=200 --no-location"
|
||||
MERGE_FLAGS="--update --width=200 --backup=none --no-fuzzy-matching"
|
||||
UNIQ_FLAGS="--width=200"
|
||||
DRY_RUN=false
|
||||
|
||||
if [[ "${1:-}" == "--dry-run" ]]; then
|
||||
DRY_RUN=true
|
||||
fi
|
||||
|
||||
info() {
|
||||
local message="$1"
|
||||
|
|
@ -84,42 +80,6 @@ function init_environment {
|
|||
make -f Makefile-docker install_node_js
|
||||
}
|
||||
|
||||
function run_l10n_extraction {
|
||||
python3 manage.py extract_content_strings
|
||||
PYTHONPATH=. DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} pybabel extract -F babel.cfg -o locale/templates/LC_MESSAGES/django.pot -c 'L10n:' -w 80 --version=1.0 --project=addons-server --copyright-holder=Mozilla .
|
||||
PYTHONPATH=. DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} pybabel extract -F babeljs.cfg -o locale/templates/LC_MESSAGES/djangojs.pot -c 'L10n:' -w 80 --version=1.0 --project=addons-server --copyright-holder=Mozilla .
|
||||
|
||||
pushd locale > /dev/null
|
||||
|
||||
info "Merging any new keys..."
|
||||
for i in `find . -name "django.po" | grep -v "en_US"`; do
|
||||
msguniq $UNIQ_FLAGS -o "$i" "$i"
|
||||
msgmerge $MERGE_FLAGS "$i" "templates/LC_MESSAGES/django.pot"
|
||||
done
|
||||
msgen templates/LC_MESSAGES/django.pot | msgmerge $MERGE_FLAGS en_US/LC_MESSAGES/django.po -
|
||||
|
||||
info "Merging any new javascript keys..."
|
||||
for i in `find . -name "djangojs.po" | grep -v "en_US"`; do
|
||||
msguniq $UNIQ_FLAGS -o "$i" "$i"
|
||||
msgmerge $MERGE_FLAGS "$i" "templates/LC_MESSAGES/djangojs.pot"
|
||||
done
|
||||
msgen templates/LC_MESSAGES/djangojs.pot | msgmerge $MERGE_FLAGS en_US/LC_MESSAGES/djangojs.po -
|
||||
|
||||
info "Cleaning out obsolete messages..."
|
||||
for i in `find . -name "django.po"`; do
|
||||
msgattrib $CLEAN_FLAGS --output-file=$i $i
|
||||
done
|
||||
for i in `find . -name "djangojs.po"`; do
|
||||
msgattrib $CLEAN_FLAGS --output-file=$i $i
|
||||
done
|
||||
|
||||
msgfilter -i sr/LC_MESSAGES/django.po -o sr_Latn/LC_MESSAGES/django.po recode-sr-latin
|
||||
|
||||
popd > /dev/null
|
||||
|
||||
info "Done extracting."
|
||||
}
|
||||
|
||||
function commit {
|
||||
info "Committing..."
|
||||
git -c user.name="$ROBOT_NAME" -c user.email="$ROBOT_EMAIL" commit -m "$MESSAGE" --author "$ROBOT_NAME <$ROBOT_EMAIL>" --no-gpg-sign locale/*/LC_MESSAGES/*.po locale/templates/
|
||||
|
|
@ -147,19 +107,27 @@ function create_pull_request {
|
|||
info "Pull request created."
|
||||
}
|
||||
|
||||
init_environment
|
||||
|
||||
run_l10n_extraction
|
||||
|
||||
commit
|
||||
|
||||
# This script is meant to be run inside a virtualenv or inside our docker
|
||||
# container. If it's the latter, it doesn't necessarily have access to the ssh
|
||||
# config, therefore we can't reliably push and create a pull request without a
|
||||
# GitHub API token.
|
||||
if [[ -z "${GITHUB_TOKEN-}" ]]
|
||||
then
|
||||
info "No github token present. You should now go back to your normal environment to push this branch and create the pull request."
|
||||
if [[ $DRY_RUN == true ]]; then
|
||||
info "Dry run only. Not committing."
|
||||
else
|
||||
create_pull_request
|
||||
info "This script will extract new strings and update the .po/.pot files."
|
||||
|
||||
init_environment
|
||||
fi
|
||||
|
||||
./scripts/run_l10n_extraction.sh
|
||||
|
||||
if [[ $DRY_RUN == false ]]; then
|
||||
commit
|
||||
|
||||
# This script is meant to be run inside a virtualenv or inside our docker
|
||||
# container. If it's the latter, it doesn't necessarily have access to the ssh
|
||||
# config, therefore we can't reliably push and create a pull request without a
|
||||
# GitHub API token.
|
||||
if [[ -z "${GITHUB_TOKEN-}" ]]; then
|
||||
info "No github token present. You should now go back to your normal environment to push this branch and create the pull request."
|
||||
else
|
||||
create_pull_request
|
||||
fi
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,67 @@
|
|||
#! /bin/bash
|
||||
|
||||
# Exit immediately when a command fails.
|
||||
set -e
|
||||
|
||||
# Make sure exit code are respected in a pipeline.
|
||||
set -o pipefail
|
||||
|
||||
# Treat unset variables as an error an exit immediately.
|
||||
set -u
|
||||
|
||||
# Extraction needs our django settings for jinja, so we need a django settings
|
||||
# module set. Since this command is meant to be run in local envs, we use
|
||||
# "settings".
|
||||
DJANGO_SETTINGS_MODULE=settings
|
||||
|
||||
# gettext flags
|
||||
CLEAN_FLAGS="--no-obsolete --width=200 --no-location"
|
||||
MERGE_FLAGS="--update --width=200 --backup=none --no-fuzzy-matching"
|
||||
UNIQ_FLAGS="--width=200"
|
||||
|
||||
info() {
|
||||
local message="$1"
|
||||
|
||||
echo ""
|
||||
echo "INFO: $message"
|
||||
echo ""
|
||||
}
|
||||
|
||||
info "Extracting content strings..."
|
||||
python3 manage.py extract_content_strings
|
||||
|
||||
info "Extracting strings from python..."
|
||||
# We must set PYTHONPATH here because pybabel needs to be able to import our settings file from the root
|
||||
PYTHONPATH=. DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} pybabel extract -F babel.cfg -o locale/templates/LC_MESSAGES/django.pot -c 'L10n:' -w 80 --version=1.0 --project=addons-server --copyright-holder=Mozilla .
|
||||
info "Extracting strings from javascript..."
|
||||
PYTHONPATH=. DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} pybabel extract -F babeljs.cfg -o locale/templates/LC_MESSAGES/djangojs.pot -c 'L10n:' -w 80 --version=1.0 --project=addons-server --copyright-holder=Mozilla .
|
||||
|
||||
pushd locale > /dev/null
|
||||
|
||||
info "Merging any new keys from templates/LC_MESSAGES/django.pot"
|
||||
for i in `find . -name "django.po" | grep -v "en_US"`; do
|
||||
msguniq $UNIQ_FLAGS -o "$i" "$i"
|
||||
msgmerge $MERGE_FLAGS "$i" "templates/LC_MESSAGES/django.pot"
|
||||
done
|
||||
msgen templates/LC_MESSAGES/django.pot | msgmerge $MERGE_FLAGS en_US/LC_MESSAGES/django.po -
|
||||
|
||||
info "Merging any new keys from templates/LC_MESSAGES/djangojs.pot"
|
||||
for i in `find . -name "djangojs.po" | grep -v "en_US"`; do
|
||||
msguniq $UNIQ_FLAGS -o "$i" "$i"
|
||||
msgmerge $MERGE_FLAGS "$i" "templates/LC_MESSAGES/djangojs.pot"
|
||||
done
|
||||
msgen templates/LC_MESSAGES/djangojs.pot | msgmerge $MERGE_FLAGS en_US/LC_MESSAGES/djangojs.po -
|
||||
|
||||
info "Cleaning out obsolete messages..."
|
||||
for i in `find . -name "django.po"`; do
|
||||
msgattrib $CLEAN_FLAGS --output-file=$i $i
|
||||
done
|
||||
for i in `find . -name "djangojs.po"`; do
|
||||
msgattrib $CLEAN_FLAGS --output-file=$i $i
|
||||
done
|
||||
|
||||
msgfilter -i sr/LC_MESSAGES/django.po -o sr_Latn/LC_MESSAGES/django.po recode-sr-latin
|
||||
|
||||
popd > /dev/null
|
||||
|
||||
info "Done extracting."
|
||||
Загрузка…
Ссылка в новой задаче