Unity-Technologies
/
electron
зеркало из https://github.com/Unity-Technologies/electron.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

ci: enable goma for all testing builds (#21992) 

* ci: enable goma for all testing builds

* Use powershell script to start goma on Windows

* Only run ensure_start from PS script.

* add missing percent

* Run stats via command instead of powershell

* use command env var

* Use out cache for faster mac builds

* Disable unneeded logging

* .git directories need to be delete before gn gen

Even though gn gen shows and error about a missing .git directory, the .git delete needs to happen before gn gen.
This commit is contained in:
John Kleinschmidt 2020-02-06 12:56:18 -05:00 коммит произвёл GitHub
Родитель 3f880fb7c7
Коммит e7982623ec
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
3 изменённых файлов: 86 добавлений и 128 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

189
.circleci/config.yml
Просмотреть файл

@ -228,15 +228,6 @@ step-setup-env-for-build: &step-setup-env-for-build
command: |
# To find `gn` executable.
echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV
# Setup Goma if applicable
if [ ! -z "$RAW_GOMA_AUTH" ] && [ "`uname`" == "Linux" ]; then
export USE_SCCACHE="false"
echo 'export USE_SCCACHE=false' >> $BASH_ENV
echo 'export USE_GOMA=true' >> $BASH_ENV
echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV
echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config
src/electron/external_binaries/goma/goma_ctl.py ensure_start
fi
if [ "$USE_SCCACHE" == "true" ]; then
# https://github.com/mozilla/sccache
@ -249,6 +240,29 @@ step-setup-env-for-build: &step-setup-env-for-build
fi
fi
step-setup-goma-for-build: &step-setup-goma-for-build
run:
name: Setup Goma
command: |
echo 'export USE_GOMA=true' >> $BASH_ENV
if [ "`uname`" == "Linux" ]; then
echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV
else
echo 'export NUMBER_OF_NINJA_PROCESSES=25' >> $BASH_ENV
fi
if [ ! -z "$RAW_GOMA_AUTH" ]; then
echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config
fi
git clone https://github.com/electron/build-tools.git
cd build-tools
npm install
mkdir third_party
node -e "require('./src/utils/goma.js').downloadAndPrepare()"
node -e "require('./src/utils/goma.js').ensure()"
echo 'export GN_GOMA_FILE='`node -e "console.log(require('./src/utils/goma.js').gnFilePath)"` >> $BASH_ENV
echo 'export LOCAL_GOMA_DIR='`node -e "console.log(require('./src/utils/goma.js').dir)"` >> $BASH_ENV
cd ..
step-restore-brew-cache: &step-restore-brew-cache
restore_cache:
paths:
@ -339,7 +353,7 @@ step-gn-gen-default: &step-gn-gen-default
command: |
cd src
if [ "$USE_GOMA" == "true" ]; then
gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"//electron/build/args/goma.gn\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
else
gn gen out/Default --args="import(\"$GN_CONFIG\") cc_wrapper=\"$SCCACHE_PATH\" $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
fi
@ -595,7 +609,7 @@ step-show-sccache-stats: &step-show-sccache-stats
$SCCACHE_PATH -s
fi
if [ "$USE_GOMA" == "true" ]; then
src/electron/external_binaries/goma/goma_ctl.py stat
$LOCAL_GOMA_DIR/goma_ctl.py stat
fi
step-mksnapshot-build: &step-mksnapshot-build
@ -964,69 +978,6 @@ steps-electron-gn-check: &steps-electron-gn-check
- *step-gn-gen-default
- *step-gn-check
steps-electron-build: &steps-electron-build
steps:
- attach_workspace:
at: .
- *step-maybe-early-exit-doc-only-change
- *step-depot-tools-add-to-path
- *step-setup-env-for-build
- *step-restore-brew-cache
- *step-get-more-space-on-mac
- *step-install-npm-deps-on-mac
- *step-fix-sync-on-mac
- *step-delete-git-directories
- *step-gn-gen-default
# Electron app
- *step-electron-build
- *step-maybe-electron-dist-strip
- *step-electron-dist-build
- *step-electron-dist-store
- *step-ninja-summary
# Native test targets
- *step-native-unittests-build
- *step-native-unittests-store
# Node.js headers
- *step-nodejs-headers-build
- *step-nodejs-headers-store
- *step-show-sccache-stats
# mksnapshot
- *step-mksnapshot-build
- *step-mksnapshot-store
- *step-maybe-cross-arch-snapshot
- *step-maybe-cross-arch-snapshot-store
# chromedriver
- *step-electron-chromedriver-gn-gen
- *step-electron-chromedriver-build
- *step-electron-chromedriver-store
# ffmpeg
- *step-ffmpeg-gn-gen
- *step-ffmpeg-build
- *step-ffmpeg-store
# hunspell
- *step-hunspell-build
- *step-hunspell-store
# Save all data needed for a further tests run.
- *step-persist-data-for-tests
- *step-maybe-generate-breakpad-symbols
- *step-maybe-zip-symbols
- *step-symbols-store
# Trigger tests on arm hardware if needed
- *step-maybe-trigger-arm-test
- *step-maybe-notify-slack-failure
steps-electron-ts-compile-for-doc-change: &steps-electron-ts-compile-for-doc-change
steps:
# Checkout - Copied ffrom steps-checkout
@ -1394,10 +1345,10 @@ commands:
steps:
- *step-depot-tools-add-to-path
- *step-setup-env-for-build
- *step-setup-goma-for-build
- *step-get-more-space-on-mac
- *step-fix-sync-on-mac
- *step-delete-git-directories
- *step-gn-gen-default
# Electron app
- when:
@ -1569,7 +1520,6 @@ jobs:
environment:
<<: *env-linux-2xlarge
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
@ -1583,7 +1533,6 @@ jobs:
environment:
<<: *env-linux-2xlarge
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
<<: *env-disable-run-as-node
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
@ -1605,7 +1554,6 @@ jobs:
environment:
<<: *env-linux-medium
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
@ -1614,10 +1562,13 @@ jobs:
environment:
<<: *env-linux-2xlarge-release
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *env-ninja-status
<<: *steps-electron-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
- electron-build:
persist: true
checkout: true
linux-x64-publish:
<<: *machine-linux-2xlarge
@ -1635,7 +1586,6 @@ jobs:
<<: *env-linux-2xlarge
<<: *env-ia32
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
@ -1650,7 +1600,6 @@ jobs:
<<: *env-linux-medium
<<: *env-ia32
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
@ -1660,10 +1609,13 @@ jobs:
<<: *env-linux-2xlarge-release
<<: *env-ia32
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *env-ninja-status
<<: *steps-electron-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
- electron-build:
persist: true
checkout: true
linux-ia32-publish:
<<: *machine-linux-2xlarge
@ -1683,7 +1635,6 @@ jobs:
<<: *env-linux-2xlarge
<<: *env-arm
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
TRIGGER_ARM_TEST: true
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
@ -1699,7 +1650,6 @@ jobs:
<<: *env-linux-medium
<<: *env-arm
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
@ -1709,10 +1659,13 @@ jobs:
<<: *env-linux-2xlarge-release
<<: *env-arm
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *env-ninja-status
<<: *steps-electron-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
- electron-build:
persist: false
checkout: true
linux-arm-publish:
<<: *machine-linux-2xlarge
@ -1732,7 +1685,6 @@ jobs:
<<: *env-linux-2xlarge
<<: *env-arm64
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
TRIGGER_ARM_TEST: true
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
@ -1756,7 +1708,6 @@ jobs:
<<: *env-linux-medium
<<: *env-arm64
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
@ -1766,10 +1717,13 @@ jobs:
<<: *env-linux-2xlarge-release
<<: *env-arm64
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *env-ninja-status
<<: *steps-electron-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
- electron-build:
persist: false
checkout: true
linux-arm64-publish:
<<: *machine-linux-2xlarge
@ -1787,7 +1741,6 @@ jobs:
environment:
<<: *env-mac-large
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
<<: *env-macos-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
@ -1810,7 +1763,6 @@ jobs:
environment:
<<: *env-machine-mac
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
@ -1819,9 +1771,14 @@ jobs:
environment:
<<: *env-mac-large
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-ninja-status
<<: *steps-electron-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
steps:
- electron-build:
persist: true
checkout: false
checkout-and-assume-cache: true
attach: false
osx-publish:
<<: *machine-mac-large
@ -1839,7 +1796,6 @@ jobs:
<<: *env-mac-large
<<: *env-mas
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
<<: *env-macos-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
@ -1863,7 +1819,6 @@ jobs:
environment:
<<: *env-machine-mac
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
@ -1873,10 +1828,15 @@ jobs:
<<: *env-mac-large
<<: *env-mas
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-ninja-status
<<: *steps-electron-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
steps:
- electron-build:
persist: true
checkout: false
checkout-and-assume-cache: true
attach: false
mas-publish:
<<: *machine-mac-large
environment:
@ -1894,7 +1854,6 @@ jobs:
environment:
<<: *env-linux-2xlarge
<<: *env-unittests
<<: *env-enable-sccache
<<: *env-headless-testing
<<: *steps-native-tests
@ -1903,7 +1862,6 @@ jobs:
environment:
<<: *env-linux-2xlarge
<<: *env-unittests
<<: *env-enable-sccache
<<: *env-headless-testing
TESTS_ARGS: '--only-disabled-tests'
<<: *steps-native-tests
@ -1913,7 +1871,6 @@ jobs:
environment:
<<: *env-linux-2xlarge
<<: *env-unittests
<<: *env-enable-sccache
<<: *env-headless-testing
TESTS_ARGS: '--include-disabled-tests'
<<: *steps-native-tests
@ -1924,7 +1881,6 @@ jobs:
<<: *env-linux-2xlarge
<<: *env-browsertests
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-headless-testing
<<: *steps-native-tests
@ -2302,10 +2258,9 @@ workflows:
- *chromium-upgrade-branches
jobs:
- linux-checkout-fast
- linux-checkout-and-save-cache
- linux-x64-release:
requires:
- linux-checkout-fast
- linux-x64-release
- linux-x64-release-tests:
requires:
- linux-x64-release
@ -2329,9 +2284,7 @@ workflows:
- linux-x64-verify-ffmpeg
- linux-x64-chromedriver
- linux-ia32-release:
requires:
- linux-checkout-fast
- linux-ia32-release
- linux-ia32-release-tests:
requires:
- linux-ia32-release
@ -2355,9 +2308,7 @@ workflows:
- linux-ia32-verify-ffmpeg
- linux-ia32-chromedriver
- linux-arm-release:
requires:
- linux-checkout-fast
- linux-arm-release
- linux-arm-chromedriver:
requires:
- linux-checkout-fast
@ -2366,10 +2317,7 @@ workflows:
- linux-arm-release
- linux-arm-chromedriver
- linux-arm64-release:
requires:
- linux-checkout-fast
- linux-arm64-release
- linux-arm64-chromedriver:
requires:
- linux-checkout-fast
@ -2389,10 +2337,11 @@ workflows:
- *chromium-upgrade-branches
jobs:
- mac-checkout-fast
- mac-checkout-and-save-cache
- osx-release:
requires:
- mac-checkout-fast
- mac-checkout-and-save-cache
- osx-release-tests:
requires:
- osx-release
@ -2418,7 +2367,7 @@ workflows:
- mas-release:
requires:
- mac-checkout-fast
- mac-checkout-and-save-cache
- mas-release-tests:
requires:
- mas-release

24
appveyor.yml
Просмотреть файл

@ -118,19 +118,29 @@ build_script:
}
}
- ps: >-
if (Test-Path 'env:RAW_GOMA_AUTH') {
$env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config"
$env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE
.\src\electron\script\start-goma.ps1 -gomaDir "$pwd\src\electron\external_binaries\goma"
if ($env:GN_CONFIG -ne 'release') {
if (Test-Path 'env:RAW_GOMA_AUTH') {
$env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config"
$env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE
}
git clone https://github.com/electron/build-tools.git
cd build-tools
npm install
mkdir third_party
node -e "require('./src/utils/goma.js').downloadAndPrepare()"
$env:GN_GOMA_FILE = node -e "console.log(require('./src/utils/goma.js').gnFilePath)"
$env:LOCAL_GOMA_DIR = node -e "console.log(require('./src/utils/goma.js').dir)"
cd ..
.\src\electron\script\start-goma.ps1 -gomaDir $env:LOCAL_GOMA_DIR
}
- cd src
- set BUILD_CONFIG_PATH=//electron/build/args/%GN_CONFIG%.gn
- if DEFINED RAW_GOMA_AUTH (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"//electron/build/args/goma.gn\") %GN_EXTRA_ARGS% ") else (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") %GN_EXTRA_ARGS% cc_wrapper=\"%SCCACHE_PATH%\"")
- if DEFINED GN_GOMA_FILE (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"%GN_GOMA_FILE%\") %GN_EXTRA_ARGS% ") else (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") %GN_EXTRA_ARGS% cc_wrapper=\"%SCCACHE_PATH%\"")
- gn check out/Default //electron:electron_lib
- gn check out/Default //electron:electron_app
- gn check out/Default //electron:manifests
- gn check out/Default //electron/shell/common/api:mojo
- if DEFINED RAW_GOMA_AUTH (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app)
- if DEFINED GN_GOMA_FILE (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app)
- if "%GN_CONFIG%"=="testing" ( python C:\depot_tools\post_build_ninja_summary.py -C out\Default )
- gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") %GN_EXTRA_ARGS%"
- ninja -C out/ffmpeg electron:electron_ffmpeg_zip
@ -140,7 +150,7 @@ build_script:
- ninja -C out/Default electron:hunspell_dictionaries_zip
- ninja -C out/Default electron:electron_chromedriver_zip
- ninja -C out/Default third_party/electron_node:headers
- if DEFINED RAW_GOMA_AUTH (python electron\external_binaries\goma\goma_ctl.py stat)
- if "%GN_CONFIG%"=="testing" ( python %LOCAL_GOMA_DIR%\goma_ctl.py stat )
- python electron/build/profile_toolchain.py --output-json=out/Default/windows_toolchain_profile.json
- appveyor PushArtifact out/Default/windows_toolchain_profile.json
- appveyor PushArtifact out/Default/dist.zip

1
script/start-goma.ps1
Просмотреть файл

@ -1,7 +1,6 @@
param([string]$gomaDir=$PWD)
$cmdPath = Join-Path -Path $gomaDir -ChildPath "goma_ctl.py"
Start-Process -FilePath cmd -ArgumentList "/C", "python", "$cmdPath", "ensure_start"
$env:GLOG_logtostderr = "true"
$timedOut = $false; $waitTime = 0; $waitIncrement = 5; $timeout=120;
Do { sleep $waitIncrement; $timedOut = (($waitTime+=$waitIncrement) -gt $timeout); iex "$gomaDir\gomacc.exe port 2" > $null; } Until(($LASTEXITCODE -eq 0) -or $timedOut)
if ($timedOut) {