From 43f36b5b244879272846594d1d138b83dee830a5 Mon Sep 17 00:00:00 2001 From: Keeley Hammond Date: Mon, 29 Nov 2021 15:09:07 -0800 Subject: [PATCH] build: add CI path-filtering for docs-only changes (#31741) build: (wip) initial dynamic config research * build: (wip) test path filtering option * build: (wip) remove doc-only script, use path filtering to check changes * build: (wip) add docker image with Electron dependencies * build: (wip) clean up config * build (wip): readd parameters, executors and env*s * build: re-add steps and commands * build: change doc-only to ts-compile-doc-only * build: re-add workflows and jobs * build: split configs to setup & build * build: move lint to "always run" config * build: clean up, remove old reference config * build: bump to path-filtering 0.1.0 * build: remove ts-compile step from build-linux * build: remove nightly-linux-release-test, linux-checks-nightly * build: don't run build on publish * build: set base-revision to main (runs branch vs commit) * build: update config from chromium roll --- .circleci/build_config.yml | 2413 ++++++++++++++++++++++++++ .circleci/config.yml | 2526 +--------------------------- script/release/ci-release-build.js | 6 +- 3 files changed, 2449 insertions(+), 2496 deletions(-) create mode 100644 .circleci/build_config.yml diff --git a/.circleci/build_config.yml b/.circleci/build_config.yml new file mode 100644 index 0000000000..f53bb9a7d1 --- /dev/null +++ b/.circleci/build_config.yml @@ -0,0 +1,2413 @@ +version: 2.1 + +parameters: + run-docs-only: + type: boolean + default: false + + upload-to-s3: + type: string + default: '1' + + run-build-linux: + type: boolean + default: false + + run-build-mac: + type: boolean + default: false + + run-linux-publish: + type: boolean + default: false + + linux-publish-arch-limit: + type: enum + default: all + enum: ["all", "arm", "arm64", "x64", "ia32"] + + run-macos-publish: + type: boolean + default: false + + macos-publish-arch-limit: + type: enum + default: all + enum: ["all", "osx-x64", "osx-arm64", "mas-x64", "mas-arm64"] + +# Executors +executors: + linux-docker: + parameters: + size: + description: "Docker executor size" + default: 2xlarge+ + type: enum + enum: ["medium", "xlarge", "2xlarge+"] + docker: + - image: ghcr.io/electron/build:27db4a3e3512bfd2e47f58cea69922da0835f1d9 + resource_class: << parameters.size >> + + macos: + parameters: + size: + description: "macOS executor size" + default: large + type: enum + enum: ["medium", "large"] + macos: + xcode: "12.4.0" + resource_class: << parameters.size >> + + # Electron Runners + apple-silicon: + resource_class: electronjs/macos-arm64 + machine: true + + linux-arm: + resource_class: electronjs/linux-arm + machine: true + + linux-arm64: + resource_class: electronjs/linux-arm64 + machine: true + +# The config expects the following environment variables to be set: +# - "SLACK_WEBHOOK" Slack hook URL to send notifications. +# +# The publishing scripts expect access tokens to be defined as env vars, +# but those are not covered here. +# +# CircleCI docs on variables: +# https://circleci.com/docs/2.0/env-vars/ + +# Build configurations options. +env-testing-build: &env-testing-build + GN_CONFIG: //electron/build/args/testing.gn + CHECK_DIST_MANIFEST: '1' + +env-release-build: &env-release-build + GN_CONFIG: //electron/build/args/release.gn + STRIP_BINARIES: true + GENERATE_SYMBOLS: true + CHECK_DIST_MANIFEST: '1' + IS_RELEASE: true + +env-headless-testing: &env-headless-testing + DISPLAY: ':99.0' + +env-stack-dumping: &env-stack-dumping + ELECTRON_ENABLE_STACK_DUMPING: '1' + +env-browsertests: &env-browsertests + GN_CONFIG: //electron/build/args/native_tests.gn + BUILD_TARGET: electron/spec:chromium_browsertests + TESTS_CONFIG: src/electron/spec/configs/browsertests.yml + +env-unittests: &env-unittests + GN_CONFIG: //electron/build/args/native_tests.gn + BUILD_TARGET: electron/spec:chromium_unittests + TESTS_CONFIG: src/electron/spec/configs/unittests.yml + +# Build targets options. +env-ia32: &env-ia32 + GN_EXTRA_ARGS: 'target_cpu = "x86"' + NPM_CONFIG_ARCH: ia32 + TARGET_ARCH: ia32 + +env-arm: &env-arm + GN_EXTRA_ARGS: 'target_cpu = "arm"' + MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm + BUILD_NATIVE_MKSNAPSHOT: 1 + TARGET_ARCH: arm + +env-apple-silicon: &env-apple-silicon + GN_EXTRA_ARGS: 'target_cpu = "arm64" use_prebuilt_v8_context_snapshot = true' + TARGET_ARCH: arm64 + USE_PREBUILT_V8_CONTEXT_SNAPSHOT: 1 + npm_config_arch: arm64 + +env-arm64: &env-arm64 + GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false' + MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm64 + BUILD_NATIVE_MKSNAPSHOT: 1 + TARGET_ARCH: arm64 + +env-mas: &env-mas + GN_EXTRA_ARGS: 'is_mas_build = true' + MAS_BUILD: 'true' + +env-mas-apple-silicon: &env-mas-apple-silicon + GN_EXTRA_ARGS: 'target_cpu = "arm64" is_mas_build = true use_prebuilt_v8_context_snapshot = true' + MAS_BUILD: 'true' + TARGET_ARCH: arm64 + USE_PREBUILT_V8_CONTEXT_SNAPSHOT: 1 + +env-send-slack-notifications: &env-send-slack-notifications + NOTIFY_SLACK: true + +env-global: &env-global + ELECTRON_OUT_DIR: Default + +env-linux-medium: &env-linux-medium + <<: *env-global + NUMBER_OF_NINJA_PROCESSES: 3 + +env-linux-2xlarge: &env-linux-2xlarge + <<: *env-global + NUMBER_OF_NINJA_PROCESSES: 34 + +env-linux-2xlarge-release: &env-linux-2xlarge-release + <<: *env-global + NUMBER_OF_NINJA_PROCESSES: 16 + +env-machine-mac: &env-machine-mac + <<: *env-global + NUMBER_OF_NINJA_PROCESSES: 6 + +env-mac-large: &env-mac-large + <<: *env-global + NUMBER_OF_NINJA_PROCESSES: 18 + +env-mac-large-release: &env-mac-large-release + <<: *env-global + NUMBER_OF_NINJA_PROCESSES: 8 + +env-ninja-status: &env-ninja-status + NINJA_STATUS: "[%r processes, %f/%t @ %o/s : %es] " + +env-disable-run-as-node: &env-disable-run-as-node + GN_BUILDFLAG_ARGS: 'enable_run_as_node = false' + +env-32bit-release: &env-32bit-release + # Set symbol level to 1 for 32 bit releases because of https://crbug.com/648948 + GN_BUILDFLAG_ARGS: 'symbol_level = 1' + +env-macos-build: &env-macos-build + # Disable pre-compiled headers to reduce out size, only useful for rebuilds + GN_BUILDFLAG_ARGS: 'enable_precompiled_headers = false' + +# Individual (shared) steps. +step-maybe-notify-slack-failure: &step-maybe-notify-slack-failure + run: + name: Send a Slack notification on failure + command: | + if [ "$NOTIFY_SLACK" == "true" ]; then + export MESSAGE="Build failed for *<$CIRCLE_BUILD_URL|$CIRCLE_JOB>* nightly build from *$CIRCLE_BRANCH*." + curl -g -H "Content-Type: application/json" -X POST \ + -d "{\"text\": \"$MESSAGE\", \"attachments\": [{\"color\": \"#FC5C3C\",\"title\": \"$CIRCLE_JOB nightly build results\",\"title_link\": \"$CIRCLE_BUILD_URL\"}]}" $SLACK_WEBHOOK + fi + when: on_fail + +step-maybe-notify-slack-success: &step-maybe-notify-slack-success + run: + name: Send a Slack notification on success + command: | + if [ "$NOTIFY_SLACK" == "true" ]; then + export MESSAGE="Build succeeded for *<$CIRCLE_BUILD_URL|$CIRCLE_JOB>* nightly build from *$CIRCLE_BRANCH*." + curl -g -H "Content-Type: application/json" -X POST \ + -d "{\"text\": \"$MESSAGE\", \"attachments\": [{\"color\": \"good\",\"title\": \"$CIRCLE_JOB nightly build results\",\"title_link\": \"$CIRCLE_BUILD_URL\"}]}" $SLACK_WEBHOOK + fi + when: on_success + +step-maybe-cleanup-arm64-mac: &step-maybe-cleanup-arm64-mac + run: + name: Cleanup after testing + command: | + if [ "$TARGET_ARCH" == "arm64" ] &&[ "`uname`" == "Darwin" ]; then + killall Electron || echo "No Electron processes left running" + killall Safari || echo "No Safari processes left running" + rm -rf ~/Library/Application\ Support/Electron* + rm -rf ~/Library/Application\ Support/electron* + security delete-generic-password -l "Chromium Safe Storage" || echo "✓ Keychain does not contain password from tests" + security delete-generic-password -l "Electron Test Main Safe Storage" || echo "✓ Keychain does not contain password from tests" + elif [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then + XVFB=/usr/bin/Xvfb + /sbin/start-stop-daemon --stop --exec $XVFB || echo "Xvfb not running" + pkill electron || echo "electron not running" + rm -rf ~/.config/Electron* + rm -rf ~/.config/electron* + fi + + when: always + +step-checkout-electron: &step-checkout-electron + checkout: + path: src/electron + +step-depot-tools-get: &step-depot-tools-get + run: + name: Get depot tools + command: | + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git + +step-depot-tools-add-to-path: &step-depot-tools-add-to-path + run: + name: Add depot tools to PATH + command: echo 'export PATH="$PATH:'"$PWD"'/depot_tools"' >> $BASH_ENV + +step-gclient-sync: &step-gclient-sync + run: + name: Gclient sync + command: | + # If we did not restore a complete sync then we need to sync for realz + if [ ! -s "src/electron/.circle-sync-done" ]; then + gclient config \ + --name "src/electron" \ + --unmanaged \ + $GCLIENT_EXTRA_ARGS \ + "$CIRCLE_REPOSITORY_URL" + + ELECTRON_USE_THREE_WAY_MERGE_FOR_PATCHES=1 gclient sync --with_branch_heads --with_tags + if [ "$IS_RELEASE" != "true" ]; then + # Re-export all the patches to check if there were changes. + python src/electron/script/export_all_patches.py src/electron/patches/config.json + cd src/electron + git update-index --refresh || true + if ! git diff-index --quiet HEAD --; then + # There are changes to the patches. Make a git commit with the updated patches + git add patches + GIT_COMMITTER_NAME="PatchUp" GIT_COMMITTER_EMAIL="73610968+patchup[bot]@users.noreply.github.com" git commit -m "chore: update patches" --author="PatchUp <73610968+patchup[bot]@users.noreply.github.com>" + # Export it + mkdir -p ../../patches + git format-patch -1 --stdout --keep-subject --no-stat --full-index > ../../patches/update-patches.patch + if (node ./script/push-patch.js 2> /dev/null > /dev/null); then + echo + echo "======================================================================" + echo "Changes to the patches when applying, we have auto-pushed the diff to the current branch" + echo "A new CI job will kick off shortly" + echo "======================================================================" + exit 1 + else + echo + echo "======================================================================" + echo "There were changes to the patches when applying." + echo "Check the CI artifacts for a patch you can apply to fix it." + echo "======================================================================" + exit 1 + fi + fi + fi + fi + +step-setup-env-for-build: &step-setup-env-for-build + run: + name: Setup Environment Variables + command: | + # To find `gn` executable. + echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV + +step-setup-goma-for-build: &step-setup-goma-for-build + run: + name: Setup Goma + command: | + echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV + if [ "`uname`" == "Darwin" ]; then + echo 'ulimit -n 10000' >> $BASH_ENV + echo 'sudo launchctl limit maxfiles 65536 200000' >> $BASH_ENV + fi + if [ ! -z "$RAW_GOMA_AUTH" ]; then + echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config + fi + git clone https://github.com/electron/build-tools.git + cd build-tools + npm install + mkdir third_party + node -e "require('./src/utils/goma.js').downloadAndPrepare({ gomaOneForAll: true })" + export GOMA_FALLBACK_ON_AUTH_FAILURE=true + third_party/goma/goma_ctl.py ensure_start + echo 'export GN_GOMA_FILE='`node -e "console.log(require('./src/utils/goma.js').gnFilePath)"` >> $BASH_ENV + echo 'export LOCAL_GOMA_DIR='`node -e "console.log(require('./src/utils/goma.js').dir)"` >> $BASH_ENV + echo 'export GOMA_FALLBACK_ON_AUTH_FAILURE=true' >> $BASH_ENV + cd .. + +step-restore-brew-cache: &step-restore-brew-cache + restore_cache: + paths: + - /usr/local/Cellar/gnu-tar + - /usr/local/bin/gtar + keys: + - v4-brew-cache-{{ arch }} + +step-save-brew-cache: &step-save-brew-cache + save_cache: + paths: + - /usr/local/Cellar/gnu-tar + - /usr/local/bin/gtar + key: v4-brew-cache-{{ arch }} + name: Persisting brew cache + +step-get-more-space-on-mac: &step-get-more-space-on-mac + run: + name: Free up space on MacOS + command: | + if [ "`uname`" == "Darwin" ]; then + sudo mkdir -p $TMPDIR/del-target + if [ "$TARGET_ARCH" == "arm64" ]; then + # Remount the root volume as writable, don't ask questions plz + sudo mount -uw / + fi + tmpify() { + if [ -d "$1" ]; then + sudo mv "$1" $TMPDIR/del-target/$(echo $1|shasum -a 256|head -n1|cut -d " " -f1) + fi + } + + strip_arm_deep() { + opwd=$(pwd) + cd $1 + f=$(find . -perm +111 -type f) + for fp in $f + do + if [[ $(file "$fp") == *"universal binary"* ]]; then + if [[ $(file "$fp") == *"arm64e)"* ]]; then + sudo lipo -remove arm64e "$fp" -o "$fp" || true + fi + if [[ $(file "$fp") == *"arm64)"* ]]; then + sudo lipo -remove arm64 "$fp" -o "$fp" || true + fi + fi + done + + cd $opwd + } + + tmpify /Library/Developer/CoreSimulator + tmpify ~/Library/Developer/CoreSimulator + tmpify $(xcode-select -p)/Platforms/AppleTVOS.platform + tmpify $(xcode-select -p)/Platforms/iPhoneOS.platform + tmpify $(xcode-select -p)/Platforms/WatchOS.platform + tmpify $(xcode-select -p)/Platforms/WatchSimulator.platform + tmpify $(xcode-select -p)/Platforms/AppleTVSimulator.platform + tmpify $(xcode-select -p)/Platforms/iPhoneSimulator.platform + tmpify $(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/metal/ios + tmpify $(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift + tmpify $(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.0 + tmpify ~/.rubies + tmpify ~/Library/Caches/Homebrew + tmpify /usr/local/Homebrew + sudo rm -rf $TMPDIR/del-target + + if [ "$TARGET_ARCH" == "arm64" ]; then + sudo rm -rf "/System/Library/Desktop Pictures" + sudo rm -rf /System/Library/Templates/Data + sudo rm -rf /System/Library/Speech/Voices + sudo rm -rf "/System/Library/Screen Savers" + sudo rm -rf /System/Volumes/Data/Library/Developer/CommandLineTools/SDKs + sudo rm -rf "/System/Volumes/Data/Library/Application Support/Apple/Photos/Print Products" + sudo rm -rf /System/Volumes/Data/Library/Java + sudo rm -rf /System/Volumes/Data/Library/Ruby + sudo rm -rf /System/Volumes/Data/Library/Printers + sudo rm -rf /System/iOSSupport + sudo rm -rf /System/Applications/*.app + sudo rm -rf /System/Applications/Utilities/*.app + sudo rm -rf /System/Library/LinguisticData + sudo rm -rf /System/Volumes/Data/private/var/db/dyld/* + # sudo rm -rf /System/Library/Fonts/* + # sudo rm -rf /System/Library/PreferencePanes + sudo rm -rf /System/Library/AssetsV2/* + sudo rm -rf /Applications/Safari.app + sudo rm -rf ~/project/src/build/linux + sudo rm -rf ~/project/src/third_party/catapult/tracing/test_data + sudo rm -rf ~/project/src/third_party/angle/third_party/VK-GL-CTS + + # lipo off some huge binaries arm64 versions to save space + strip_arm_deep $(xcode-select -p)/../SharedFrameworks + strip_arm_deep /System/Volumes/Data/Library/Developer/CommandLineTools/usr + fi + fi + background: true + +# On macOS delete all .git directories under src/ expect for +# third_party/angle/ because of build time generation of file +# gen/angle/commit.h depends on third_party/angle/.git/HEAD +# https://chromium-review.googlesource.com/c/angle/angle/+/2074924 +# TODO: maybe better to always leave out */.git/HEAD file for all targets ? +step-delete-git-directories: &step-delete-git-directories + run: + name: Delete all .git directories under src on MacOS to free space + command: | + if [ "`uname`" == "Darwin" ]; then + cd src + ( find . -type d -name ".git" -not -path "./third_party/angle/*" ) | xargs rm -rf + fi + +# On macOS the yarn install command during gclient sync was run on a linux +# machine and therefore installed a slightly different set of dependencies +# Notably "fsevents" is a macOS only dependency, we rerun yarn install once +# we are on a macOS machine to get the correct state +step-install-npm-deps-on-mac: &step-install-npm-deps-on-mac + run: + name: Install node_modules on MacOS + command: | + if [ "`uname`" == "Darwin" ]; then + cd src/electron + node script/yarn install + fi + +# This step handles the differences between the linux "gclient sync" +# and the expected state on macOS +step-fix-sync: &step-fix-sync + run: + name: Fix Sync + command: | + if [ "`uname`" == "Darwin" ]; then + # Fix Clang Install (wrong binary) + rm -rf src/third_party/llvm-build + python3 src/tools/clang/scripts/update.py + fi + + cd src/third_party/angle + git remote set-url origin https://chromium.googlesource.com/angle/angle.git + git fetch + +step-install-signing-cert-on-mac: &step-install-signing-cert-on-mac + run: + name: Import and trust self-signed codesigning cert on MacOS + command: | + if [ "$TARGET_ARCH" != "arm64" ] && [ "`uname`" == "Darwin" ]; then + cd src/electron + ./script/codesign/generate-identity.sh + fi + +step-install-gnutar-on-mac: &step-install-gnutar-on-mac + run: + name: Install gnu-tar on macos + command: | + if [ "`uname`" == "Darwin" ]; then + if [ ! -d /usr/local/Cellar/gnu-tar/ ]; then + brew update + brew install gnu-tar + fi + ln -fs /usr/local/bin/gtar /usr/local/bin/tar + fi + +step-gn-gen-default: &step-gn-gen-default + run: + name: Default GN gen + command: | + cd src + gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" + +step-gn-check: &step-gn-check + run: + name: GN check + command: | + cd src + gn check out/Default //electron:electron_lib + gn check out/Default //electron:electron_app + gn check out/Default //electron/shell/common/api:mojo + # Check the hunspell filenames + node electron/script/gen-hunspell-filenames.js --check + node electron/script/gen-libc++-filenames.js --check + +step-electron-build: &step-electron-build + run: + name: Electron build + no_output_timeout: 30m + command: | + # On arm platforms we generate a cross-arch ffmpeg that ninja does not seem + # to realize is not correct / should be rebuilt. We delete it here so it is + # rebuilt + if [ "$TRIGGER_ARM_TEST" == "true" ]; then + rm -f src/out/Default/libffmpeg.so + fi + cd src + # Enable if things get really bad + # if [ "$TARGET_ARCH" == "arm64" ] &&[ "`uname`" == "Darwin" ]; then + # diskutil erasevolume HFS+ "xcode_disk" `hdiutil attach -nomount ram://12582912` + # mv /Applications/Xcode-12.beta.5.app /Volumes/xcode_disk/ + # ln -s /Volumes/xcode_disk/Xcode-12.beta.5.app /Applications/Xcode-12.beta.5.app + # fi + + # Lets generate a snapshot and mksnapshot and then delete all the x-compiled generated files to save space + if [ "$USE_PREBUILT_V8_CONTEXT_SNAPSHOT" == "1" ]; then + ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES + ninja -C out/Default tools/v8_context_snapshot -j $NUMBER_OF_NINJA_PROCESSES + gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args + (cd out/Default; zip mksnapshot.zip mksnapshot_args clang_x64_v8_arm64/gen/v8/embedded.S) + rm -rf out/Default/clang_x64_v8_arm64/gen + rm -rf out/Default/clang_x64_v8_arm64/obj + rm -rf out/Default/clang_x64/obj + + # Regenerate because we just deleted some ninja files + gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" + fi + NINJA_SUMMARIZE_BUILD=1 autoninja -C out/Default electron -j $NUMBER_OF_NINJA_PROCESSES + cp out/Default/.ninja_log out/electron_ninja_log + node electron/script/check-symlinks.js + +step-native-unittests-build: &step-native-unittests-build + run: + name: Build native test targets + no_output_timeout: 30m + command: | + cd src + ninja -C out/Default shell_browser_ui_unittests -j $NUMBER_OF_NINJA_PROCESSES + +step-maybe-electron-dist-strip: &step-maybe-electron-dist-strip + run: + name: Strip electron binaries + command: | + if [ "$STRIP_BINARIES" == "true" ] && [ "`uname`" == "Linux" ]; then + if [ x"$TARGET_ARCH" == x ]; then + target_cpu=x64 + elif [ "$TARGET_ARCH" == "ia32" ]; then + target_cpu=x86 + else + target_cpu="$TARGET_ARCH" + fi + cd src + electron/script/copy-debug-symbols.py --target-cpu="$target_cpu" --out-dir=out/Default/debug --compress + electron/script/strip-binaries.py --target-cpu="$target_cpu" + electron/script/add-debug-link.py --target-cpu="$target_cpu" --debug-dir=out/Default/debug + fi + +step-electron-dist-build: &step-electron-dist-build + run: + name: Build dist.zip + command: | + cd src + if [ "$SKIP_DIST_ZIP" != "1" ]; then + ninja -C out/Default electron:electron_dist_zip + if [ "$CHECK_DIST_MANIFEST" == "1" ]; then + if [ "`uname`" == "Darwin" ]; then + target_os=mac + target_cpu=x64 + if [ x"$MAS_BUILD" == x"true" ]; then + target_os=mac_mas + fi + if [ "$TARGET_ARCH" == "arm64" ]; then + target_cpu=arm64 + fi + elif [ "`uname`" == "Linux" ]; then + target_os=linux + if [ x"$TARGET_ARCH" == x ]; then + target_cpu=x64 + elif [ "$TARGET_ARCH" == "ia32" ]; then + target_cpu=x86 + else + target_cpu="$TARGET_ARCH" + fi + else + echo "Unknown system: `uname`" + exit 1 + fi + electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.$target_cpu.manifest + fi + fi + +step-electron-chromedriver-build: &step-electron-chromedriver-build + run: + name: Build chromedriver.zip + command: | + cd src + if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then + gn gen out/chromedriver --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") is_component_ffmpeg=false proprietary_codecs=false $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" + export CHROMEDRIVER_DIR="out/chromedriver" + else + export CHROMEDRIVER_DIR="out/Default" + fi + ninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver -j $NUMBER_OF_NINJA_PROCESSES + if [ "`uname`" == "Linux" ]; then + electron/script/strip-binaries.py --target-cpu="$TARGET_ARCH" --file $PWD/$CHROMEDRIVER_DIR/chromedriver + fi + ninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver_zip + if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then + cp out/chromedriver/chromedriver.zip out/Default + fi + +step-nodejs-headers-build: &step-nodejs-headers-build + run: + name: Build Node.js headers + command: | + cd src + ninja -C out/Default third_party/electron_node:headers + +step-electron-publish: &step-electron-publish + run: + name: Publish Electron Dist + command: | + if [ "`uname`" == "Darwin" ]; then + rm -rf src/out/Default/obj + fi + + cd src/electron + if [ "$UPLOAD_TO_S3" == "1" ]; then + echo 'Uploading Electron release distribution to S3' + script/release/uploaders/upload.py --verbose --upload_to_s3 + else + echo 'Uploading Electron release distribution to Github releases' + script/release/uploaders/upload.py --verbose + fi + +step-persist-data-for-tests: &step-persist-data-for-tests + persist_to_workspace: + root: . + paths: + # Build artifacts + - src/out/Default/dist.zip + - src/out/Default/mksnapshot.zip + - src/out/Default/chromedriver.zip + - src/out/Default/shell_browser_ui_unittests + - src/out/Default/gen/node_headers + - src/out/ffmpeg/ffmpeg.zip + - src/electron + - src/third_party/electron_node + - src/third_party/nan + - src/cross-arch-snapshots + - src/third_party/llvm-build + - src/build/linux + - src/buildtools/third_party/libc++ + - src/buildtools/third_party/libc++abi + - src/out/Default/obj/buildtools/third_party + +step-electron-dist-unzip: &step-electron-dist-unzip + run: + name: Unzip dist.zip + command: | + cd src/out/Default + # -o overwrite files WITHOUT prompting + # TODO(alexeykuzmin): Remove '-o' when it's no longer needed. + # -: allows to extract archive members into locations outside + # of the current ``extraction root folder''. + # ASan builds have the llvm-symbolizer binaries listed as + # runtime_deps, with their paths as `../../third_party/...` + # unzip exits with non-zero code on such zip files unless -: is + # passed. + unzip -:o dist.zip + +step-ffmpeg-unzip: &step-ffmpeg-unzip + run: + name: Unzip ffmpeg.zip + command: | + cd src/out/ffmpeg + unzip -:o ffmpeg.zip + +step-mksnapshot-unzip: &step-mksnapshot-unzip + run: + name: Unzip mksnapshot.zip + command: | + cd src/out/Default + unzip -:o mksnapshot.zip + +step-chromedriver-unzip: &step-chromedriver-unzip + run: + name: Unzip chromedriver.zip + command: | + cd src/out/Default + unzip -:o chromedriver.zip + +step-ffmpeg-gn-gen: &step-ffmpeg-gn-gen + run: + name: ffmpeg GN gen + command: | + cd src + gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS" + +step-ffmpeg-build: &step-ffmpeg-build + run: + name: Non proprietary ffmpeg build + command: | + cd src + ninja -C out/ffmpeg electron:electron_ffmpeg_zip -j $NUMBER_OF_NINJA_PROCESSES + +step-verify-ffmpeg: &step-verify-ffmpeg + run: + name: Verify ffmpeg + command: | + cd src + python electron/script/verify-ffmpeg.py --source-root "$PWD" --build-dir out/Default --ffmpeg-path out/ffmpeg + +step-verify-mksnapshot: &step-verify-mksnapshot + run: + name: Verify mksnapshot + command: | + if [ "$IS_ASAN" != "1" ]; then + cd src + if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then + python electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default --snapshot-files-dir $PWD/cross-arch-snapshots + else + python electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default + fi + fi + +step-verify-chromedriver: &step-verify-chromedriver + run: + name: Verify ChromeDriver + command: | + if [ "$IS_ASAN" != "1" ]; then + cd src + python electron/script/verify-chromedriver.py --source-root "$PWD" --build-dir out/Default + fi + +step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing + run: + name: Setup for headless testing + command: | + if [ "`uname`" != "Darwin" ]; then + sh -e /etc/init.d/xvfb start + fi + +step-show-goma-stats: &step-show-goma-stats + run: + shell: /bin/bash + name: Check goma stats after build + command: | + set +e + set +o pipefail + $LOCAL_GOMA_DIR/goma_ctl.py stat + $LOCAL_GOMA_DIR/diagnose_goma_log.py + true + when: always + +step-mksnapshot-build: &step-mksnapshot-build + run: + name: mksnapshot build + command: | + cd src + if [ "$USE_PREBUILT_V8_CONTEXT_SNAPSHOT" != "1" ]; then + ninja -C out/Default electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES + gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args + fi + if [ "`uname`" != "Darwin" ]; then + if [ "$TARGET_ARCH" == "arm" ]; then + electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/mksnapshot + electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/v8_context_snapshot_generator + elif [ "$TARGET_ARCH" == "arm64" ]; then + electron/script/strip-binaries.py --file $PWD/out/Default/clang_x64_v8_arm64/mksnapshot + electron/script/strip-binaries.py --file $PWD/out/Default/clang_x64_v8_arm64/v8_context_snapshot_generator + else + electron/script/strip-binaries.py --file $PWD/out/Default/mksnapshot + electron/script/strip-binaries.py --file $PWD/out/Default/v8_context_snapshot_generator + fi + fi + if [ "$USE_PREBUILT_V8_CONTEXT_SNAPSHOT" != "1" ] && [ "$SKIP_DIST_ZIP" != "1" ]; then + ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES + (cd out/Default; zip mksnapshot.zip mksnapshot_args gen/v8/embedded.S) + fi + +step-hunspell-build: &step-hunspell-build + run: + name: hunspell build + command: | + cd src + if [ "$SKIP_DIST_ZIP" != "1" ]; then + ninja -C out/Default electron:hunspell_dictionaries_zip -j $NUMBER_OF_NINJA_PROCESSES + fi + +step-maybe-generate-libcxx: &step-maybe-generate-libcxx + run: + name: maybe generate libcxx + command: | + cd src + if [ "`uname`" == "Linux" ]; then + ninja -C out/Default electron:libcxx_headers_zip -j $NUMBER_OF_NINJA_PROCESSES + ninja -C out/Default electron:libcxxabi_headers_zip -j $NUMBER_OF_NINJA_PROCESSES + ninja -C out/Default electron:libcxx_objects_zip -j $NUMBER_OF_NINJA_PROCESSES + fi + +step-maybe-generate-breakpad-symbols: &step-maybe-generate-breakpad-symbols + run: + name: Generate breakpad symbols + no_output_timeout: 30m + command: | + if [ "$GENERATE_SYMBOLS" == "true" ]; then + cd src + ninja -C out/Default electron:electron_symbols + fi + +step-maybe-zip-symbols: &step-maybe-zip-symbols + run: + name: Zip symbols + command: | + cd src + export BUILD_PATH="$PWD/out/Default" + ninja -C out/Default electron:licenses + ninja -C out/Default electron:electron_version + DELETE_DSYMS_AFTER_ZIP=1 electron/script/zip-symbols.py -b $BUILD_PATH + +step-maybe-cross-arch-snapshot: &step-maybe-cross-arch-snapshot + run: + name: Generate cross arch snapshot (arm/arm64) + command: | + if [ "$GENERATE_CROSS_ARCH_SNAPSHOT" == "true" ] && [ -z "$CIRCLE_PR_NUMBER" ]; then + cd src + if [ "$TARGET_ARCH" == "arm" ]; then + export MKSNAPSHOT_PATH="clang_x86_v8_arm" + elif [ "$TARGET_ARCH" == "arm64" ]; then + export MKSNAPSHOT_PATH="clang_x64_v8_arm64" + fi + cp "out/Default/$MKSNAPSHOT_PATH/mksnapshot" out/Default + cp "out/Default/$MKSNAPSHOT_PATH/v8_context_snapshot_generator" out/Default + if [ "`uname`" == "Linux" ]; then + cp "out/Default/$MKSNAPSHOT_PATH/libffmpeg.so" out/Default + elif [ "`uname`" == "Darwin" ]; then + cp "out/Default/$MKSNAPSHOT_PATH/libffmpeg.dylib" out/Default + fi + python electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default --create-snapshot-only + mkdir cross-arch-snapshots + cp out/Default-mksnapshot-test/*.bin cross-arch-snapshots + fi + +step-maybe-generate-typescript-defs: &step-maybe-generate-typescript-defs + run: + name: Generate type declarations + command: | + if [ "`uname`" == "Darwin" ]; then + cd src/electron + node script/yarn create-typescript-definitions + fi + +step-fix-known-hosts-linux: &step-fix-known-hosts-linux + run: + name: Fix Known Hosts on Linux + command: | + if [ "`uname`" == "Linux" ]; then + ./src/electron/.circleci/fix-known-hosts.sh + fi + +# Checkout Steps +step-generate-deps-hash: &step-generate-deps-hash + run: + name: Generate DEPS Hash + command: node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target + +step-touch-sync-done: &step-touch-sync-done + run: + name: Touch Sync Done + command: touch src/electron/.circle-sync-done + +# Restore exact src cache based on the hash of DEPS and patches/* +# If no cache is matched EXACTLY then the .circle-sync-done file is empty +# If a cache is matched EXACTLY then the .circle-sync-done file contains "done" +step-maybe-restore-src-cache: &step-maybe-restore-src-cache + restore_cache: + keys: + - v8-src-cache-{{ checksum "src/electron/.depshash" }} + name: Restoring src cache +step-maybe-restore-src-cache-marker: &step-maybe-restore-src-cache-marker + restore_cache: + keys: + - v1-src-cache-marker-{{ checksum "src/electron/.depshash" }} + name: Restoring src cache marker + +# Restore exact or closest git cache based on the hash of DEPS and .circle-sync-done +# If the src cache was restored above then this will match an empty cache +# If the src cache was not restored above then this will match a close git cache +step-maybe-restore-git-cache: &step-maybe-restore-git-cache + restore_cache: + paths: + - ~/.gclient-cache + keys: + - v2-gclient-cache-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }} + - v2-gclient-cache-{{ checksum "src/electron/.circle-sync-done" }} + name: Conditionally restoring git cache + +step-restore-out-cache: &step-restore-out-cache + restore_cache: + paths: + - ./src/out/Default + keys: + - v9-out-cache-{{ checksum "src/electron/.depshash" }}-{{ checksum "src/electron/.depshash-target" }} + name: Restoring out cache + +step-set-git-cache-path: &step-set-git-cache-path + run: + name: Set GIT_CACHE_PATH to make gclient to use the cache + command: | + # CircleCI does not support interpolation when setting environment variables. + # https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command + echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV + +# Persist the git cache based on the hash of DEPS and .circle-sync-done +# If the src cache was restored above then this will persist an empty cache +step-save-git-cache: &step-save-git-cache + save_cache: + paths: + - ~/.gclient-cache + key: v2-gclient-cache-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }} + name: Persisting git cache + +step-save-out-cache: &step-save-out-cache + save_cache: + paths: + - ./src/out/Default + key: v9-out-cache-{{ checksum "src/electron/.depshash" }}-{{ checksum "src/electron/.depshash-target" }} + name: Persisting out cache + +step-run-electron-only-hooks: &step-run-electron-only-hooks + run: + name: Run Electron Only Hooks + command: gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]" + +step-generate-deps-hash-cleanly: &step-generate-deps-hash-cleanly + run: + name: Generate DEPS Hash + command: (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target + +# Mark the sync as done for future cache saving +step-mark-sync-done: &step-mark-sync-done + run: + name: Mark Sync Done + command: echo DONE > src/electron/.circle-sync-done + +# Minimize the size of the cache +step-minimize-workspace-size-from-checkout: &step-minimize-workspace-size-from-checkout + run: + name: Remove some unused data to avoid storing it in the workspace/cache + command: | + rm -rf src/android_webview + rm -rf src/ios/chrome + rm -rf src/third_party/blink/web_tests + rm -rf src/third_party/blink/perf_tests + rm -rf src/third_party/WebKit/LayoutTests + rm -rf third_party/electron_node/deps/openssl + rm -rf third_party/electron_node/deps/v8 + rm -rf chrome/test/data/xr/webvr_info + +# Save the src cache based on the deps hash +step-save-src-cache: &step-save-src-cache + save_cache: + paths: + - /var/portal + key: v8-src-cache-{{ checksum "/var/portal/src/electron/.depshash" }} + name: Persisting src cache +step-make-src-cache-marker: &step-make-src-cache-marker + run: + name: Making src cache marker + command: touch .src-cache-marker +step-save-src-cache-marker: &step-save-src-cache-marker + save_cache: + paths: + - .src-cache-marker + key: v1-src-cache-marker-{{ checksum "/var/portal/src/electron/.depshash" }} + +step-maybe-early-exit-no-doc-change: &step-maybe-early-exit-no-doc-change + run: + name: Shortcircuit job if change is not doc only + command: | + if [ ! -s src/electron/.skip-ci-build ]; then + circleci-agent step halt + fi + +step-ts-compile: &step-ts-compile + run: + name: Run TS/JS compile on doc only change + command: | + cd src + ninja -C out/Default electron:default_app_js -j $NUMBER_OF_NINJA_PROCESSES + ninja -C out/Default electron:electron_js2c -j $NUMBER_OF_NINJA_PROCESSES + +# List of all steps. +steps-electron-gn-check: &steps-electron-gn-check + steps: + - attach_workspace: + at: . + - *step-depot-tools-add-to-path + - *step-setup-env-for-build + - *step-setup-goma-for-build + - *step-gn-gen-default + - *step-gn-check + +steps-electron-ts-compile-for-doc-change: &steps-electron-ts-compile-for-doc-change + steps: + # Checkout - Copied from steps-checkout + - *step-checkout-electron + - *step-depot-tools-get + - *step-depot-tools-add-to-path + - *step-restore-brew-cache + - *step-install-gnutar-on-mac + - *step-get-more-space-on-mac + - *step-generate-deps-hash + - *step-touch-sync-done + - maybe-restore-portaled-src-cache + - *step-maybe-restore-git-cache + - *step-set-git-cache-path + # This sync call only runs if .circle-sync-done is an EMPTY file + - *step-gclient-sync + # These next few steps reset Electron to the correct commit regardless of which cache was restored + - run: + name: Wipe Electron + command: rm -rf src/electron + - *step-checkout-electron + - *step-run-electron-only-hooks + - *step-generate-deps-hash-cleanly + - *step-mark-sync-done + - *step-minimize-workspace-size-from-checkout + + - *step-depot-tools-add-to-path + - *step-setup-env-for-build + - *step-setup-goma-for-build + - *step-get-more-space-on-mac + - *step-install-npm-deps-on-mac + - *step-fix-sync + - *step-gn-gen-default + + #Compile ts/js to verify doc change didn't break anything + - *step-ts-compile + +steps-native-tests: &steps-native-tests + steps: + - attach_workspace: + at: . + - *step-depot-tools-add-to-path + - *step-setup-env-for-build + - *step-setup-goma-for-build + - *step-gn-gen-default + + - run: + name: Build tests + command: | + cd src + ninja -C out/Default $BUILD_TARGET + - *step-show-goma-stats + + - *step-setup-linux-for-headless-testing + - run: + name: Run tests + command: | + mkdir test_results + python src/electron/script/native-tests.py run \ + --config $TESTS_CONFIG \ + --tests-dir src/out/Default \ + --output-dir test_results \ + $TESTS_ARGS + + - store_artifacts: + path: test_results + destination: test_results # Put it in the root folder. + - store_test_results: + path: test_results + +steps-verify-ffmpeg: &steps-verify-ffmpeg + steps: + - attach_workspace: + at: . + - *step-depot-tools-add-to-path + - *step-electron-dist-unzip + - *step-ffmpeg-unzip + - *step-setup-linux-for-headless-testing + + - *step-verify-ffmpeg + - *step-maybe-notify-slack-failure + +steps-tests: &steps-tests + steps: + - attach_workspace: + at: . + - *step-depot-tools-add-to-path + - *step-electron-dist-unzip + - *step-mksnapshot-unzip + - *step-chromedriver-unzip + - *step-setup-linux-for-headless-testing + - *step-restore-brew-cache + - *step-fix-known-hosts-linux + - *step-install-signing-cert-on-mac + + - run: + name: Run Electron tests + environment: + MOCHA_REPORTER: mocha-multi-reporters + ELECTRON_TEST_RESULTS_DIR: junit + MOCHA_MULTI_REPORTERS: mocha-junit-reporter, tap + ELECTRON_DISABLE_SECURITY_WARNINGS: 1 + command: | + cd src + if [ "$IS_ASAN" == "1" ]; then + ASAN_SYMBOLIZE="$PWD/tools/valgrind/asan/asan_symbolize.py --executable-path=$PWD/out/Default/electron" + export ASAN_OPTIONS="symbolize=0 handle_abort=1" + export G_SLICE=always-malloc + export NSS_DISABLE_ARENA_FREE_LIST=1 + export NSS_DISABLE_UNLOAD=1 + export LLVM_SYMBOLIZER_PATH=$PWD/third_party/llvm-build/Release+Asserts/bin/llvm-symbolizer + export MOCHA_TIMEOUT=180000 + echo "Piping output to ASAN_SYMBOLIZE ($ASAN_SYMBOLIZE)" + (cd electron && node script/yarn test --runners=main --trace-uncaught --enable-logging --files $(circleci tests glob spec-main/*-spec.ts | circleci tests split --split-by=timings)) 2>&1 | $ASAN_SYMBOLIZE + (cd electron && node script/yarn test --runners=remote --trace-uncaught --enable-logging --files $(circleci tests glob spec/*-spec.js | circleci tests split --split-by=timings)) 2>&1 | $ASAN_SYMBOLIZE + else + if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then + export ELECTRON_SKIP_NATIVE_MODULE_TESTS=true + (cd electron && node script/yarn test --runners=main --trace-uncaught --enable-logging) + (cd electron && node script/yarn test --runners=remote --trace-uncaught --enable-logging) + else + if [ "$TARGET_ARCH" == "ia32" ]; then + npm_config_arch=x64 node electron/node_modules/dugite/script/download-git.js + fi + (cd electron && node script/yarn test --runners=main --trace-uncaught --enable-logging --files $(circleci tests glob spec-main/*-spec.ts | circleci tests split --split-by=timings)) + (cd electron && node script/yarn test --runners=remote --trace-uncaught --enable-logging --files $(circleci tests glob spec/*-spec.js | circleci tests split --split-by=timings)) + fi + fi + - run: + name: Check test results existence + command: | + cd src + + # Check if test results exist and are not empty. + if [ ! -s "junit/test-results-remote.xml" ]; then + exit 1 + fi + if [ ! -s "junit/test-results-main.xml" ]; then + exit 1 + fi + - store_test_results: + path: src/junit + + - *step-verify-mksnapshot + - *step-verify-chromedriver + + - *step-maybe-notify-slack-failure + + - *step-maybe-cleanup-arm64-mac + +steps-test-nan: &steps-test-nan + steps: + - attach_workspace: + at: . + - *step-depot-tools-add-to-path + - *step-electron-dist-unzip + - *step-setup-linux-for-headless-testing + - *step-fix-known-hosts-linux + - run: + name: Run Nan Tests + command: | + cd src + node electron/script/nan-spec-runner.js + +steps-test-node: &steps-test-node + steps: + - attach_workspace: + at: . + - *step-depot-tools-add-to-path + - *step-electron-dist-unzip + - *step-setup-linux-for-headless-testing + - *step-fix-known-hosts-linux + - run: + name: Run Node Tests + command: | + cd src + node electron/script/node-spec-runner.js --default --jUnitDir=junit + - store_test_results: + path: src/junit + +# Command Aliases +commands: + maybe-restore-portaled-src-cache: + parameters: + halt-if-successful: + type: boolean + default: false + steps: + - run: + name: Prepare for cross-OS sync restore + command: | + sudo mkdir -p /var/portal + sudo chown -R $(id -u):$(id -g) /var/portal + - when: + condition: << parameters.halt-if-successful >> + steps: + - *step-maybe-restore-src-cache-marker + - run: + name: Halt the job early if the src cache exists + command: | + if [ -f ".src-cache-marker" ]; then + circleci-agent step halt + fi + - *step-maybe-restore-src-cache + - run: + name: Fix the src cache restore point on macOS + command: | + if [ -d "/var/portal/src" ]; then + echo Relocating Cache + rm -rf src + mv /var/portal/src ./ + fi + + move_and_store_all_artifacts: + steps: + - run: + name: Move all generated artifacts to upload folder + command: | + rm -rf generated_artifacts + mkdir generated_artifacts + mv_if_exist() { + if [ -f "$1" ] || [ -d "$1" ]; then + echo Storing $1 + mv $1 generated_artifacts + else + echo Skipping $1 - It is not present on disk + fi + } + mv_if_exist src/out/Default/dist.zip + mv_if_exist src/out/Default/shell_browser_ui_unittests + mv_if_exist src/out/Default/gen/node_headers.tar.gz + mv_if_exist src/out/Default/symbols.zip + mv_if_exist src/out/Default/mksnapshot.zip + mv_if_exist src/out/Default/chromedriver.zip + mv_if_exist src/out/ffmpeg/ffmpeg.zip + mv_if_exist src/out/Default/hunspell_dictionaries.zip + mv_if_exist src/cross-arch-snapshots + mv_if_exist src/out/electron_ninja_log + when: always + - store_artifacts: + path: generated_artifacts + destination: ./ + - store_artifacts: + path: generated_artifacts/cross-arch-snapshots + destination: cross-arch-snapshots + + checkout-from-cache: + steps: + - *step-checkout-electron + - *step-depot-tools-get + - *step-depot-tools-add-to-path + - *step-generate-deps-hash + - maybe-restore-portaled-src-cache + - run: + name: Ensure src checkout worked + command: | + if [ ! -d "src/third_party/blink" ]; then + echo src cache was not restored for some reason, idk what happened here... + exit 1 + fi + - run: + name: Wipe Electron + command: rm -rf src/electron + - *step-checkout-electron + - *step-run-electron-only-hooks + - *step-generate-deps-hash-cleanly + electron-build: + parameters: + attach: + type: boolean + default: false + persist: + type: boolean + default: true + persist-checkout: + type: boolean + default: false + checkout: + type: boolean + default: true + checkout-and-assume-cache: + type: boolean + default: false + save-git-cache: + type: boolean + default: false + checkout-to-create-src-cache: + type: boolean + default: false + build: + type: boolean + default: true + use-out-cache: + type: boolean + default: true + restore-src-cache: + type: boolean + default: true + build-nonproprietary-ffmpeg: + type: boolean + default: true + steps: + - when: + condition: << parameters.attach >> + steps: + - attach_workspace: + at: . + - run: rm -rf src/electron + - *step-restore-brew-cache + - *step-install-gnutar-on-mac + - *step-save-brew-cache + - when: + condition: << parameters.checkout-and-assume-cache >> + steps: + - checkout-from-cache + - when: + condition: << parameters.checkout >> + steps: + # Checkout - Copied from steps-checkout + - *step-checkout-electron + - *step-depot-tools-get + - *step-depot-tools-add-to-path + - *step-get-more-space-on-mac + - *step-generate-deps-hash + - *step-touch-sync-done + - when: + condition: << parameters.restore-src-cache >> + steps: + - maybe-restore-portaled-src-cache: + halt-if-successful: << parameters.checkout-to-create-src-cache >> + - *step-maybe-restore-git-cache + - *step-set-git-cache-path + # This sync call only runs if .circle-sync-done is an EMPTY file + - *step-gclient-sync + - store_artifacts: + path: patches + - when: + condition: << parameters.save-git-cache >> + steps: + - *step-save-git-cache + # These next few steps reset Electron to the correct commit regardless of which cache was restored + - run: + name: Wipe Electron + command: rm -rf src/electron + - *step-checkout-electron + - *step-run-electron-only-hooks + - *step-generate-deps-hash-cleanly + - *step-mark-sync-done + - *step-minimize-workspace-size-from-checkout + - *step-delete-git-directories + - when: + condition: << parameters.persist-checkout >> + steps: + - persist_to_workspace: + root: . + paths: + - depot_tools + - src + - when: + condition: << parameters.checkout-to-create-src-cache >> + steps: + - run: + name: Move src folder to the cross-OS portal + command: | + sudo mkdir -p /var/portal + sudo chown -R $(id -u):$(id -g) /var/portal + mv ./src /var/portal + - *step-save-src-cache + - *step-make-src-cache-marker + - *step-save-src-cache-marker + + - when: + condition: << parameters.build >> + steps: + - *step-depot-tools-add-to-path + - *step-setup-env-for-build + - *step-setup-goma-for-build + - *step-get-more-space-on-mac + - *step-fix-sync + - *step-delete-git-directories + + # Electron app + - when: + condition: << parameters.use-out-cache >> + steps: + - *step-restore-out-cache + - *step-gn-gen-default + - *step-electron-build + - *step-maybe-electron-dist-strip + - *step-electron-dist-build + + # Native test targets + - *step-native-unittests-build + + # Node.js headers + - *step-nodejs-headers-build + + - *step-show-goma-stats + + # mksnapshot + - *step-mksnapshot-build + - *step-maybe-cross-arch-snapshot + + # chromedriver + - *step-electron-chromedriver-build + + - when: + condition: << parameters.build-nonproprietary-ffmpeg >> + steps: + # ffmpeg + - *step-ffmpeg-gn-gen + - *step-ffmpeg-build + + # hunspell + - *step-hunspell-build + + # Save all data needed for a further tests run. + - when: + condition: << parameters.persist >> + steps: + - *step-persist-data-for-tests + + - when: + condition: << parameters.build >> + steps: + - *step-maybe-generate-breakpad-symbols + - *step-maybe-zip-symbols + + - when: + condition: << parameters.build >> + steps: + - move_and_store_all_artifacts + - run: + name: Remove the big things on macOS, this seems to be better on average + command: | + if [ "`uname`" == "Darwin" ]; then + mkdir -p src/out/Default + cd src/out/Default + find . -type f -size +50M -delete + mkdir -p gen/electron + cd gen/electron + # These files do not seem to like being in a cache, let us remove them + find . -type f -name '*_pkg_info' -delete + fi + - when: + condition: << parameters.use-out-cache >> + steps: + - *step-save-out-cache + + - *step-maybe-notify-slack-failure + + electron-publish: + parameters: + attach: + type: boolean + default: false + checkout: + type: boolean + default: true + steps: + - when: + condition: << parameters.attach >> + steps: + - attach_workspace: + at: . + - when: + condition: << parameters.checkout >> + steps: + - *step-depot-tools-get + - *step-depot-tools-add-to-path + - *step-restore-brew-cache + - *step-get-more-space-on-mac + - when: + condition: << parameters.checkout >> + steps: + - *step-checkout-electron + - *step-touch-sync-done + - *step-maybe-restore-git-cache + - *step-set-git-cache-path + - *step-gclient-sync + - *step-delete-git-directories + - *step-minimize-workspace-size-from-checkout + - *step-fix-sync + - *step-setup-env-for-build + - *step-setup-goma-for-build + - *step-gn-gen-default + + # Electron app + - *step-electron-build + - *step-show-goma-stats + - *step-maybe-generate-breakpad-symbols + - *step-maybe-electron-dist-strip + - *step-electron-dist-build + - *step-maybe-zip-symbols + + # mksnapshot + - *step-mksnapshot-build + + # chromedriver + - *step-electron-chromedriver-build + + # Node.js headers + - *step-nodejs-headers-build + + # ffmpeg + - *step-ffmpeg-gn-gen + - *step-ffmpeg-build + + # hunspell + - *step-hunspell-build + + # libcxx + - *step-maybe-generate-libcxx + + # typescript defs + - *step-maybe-generate-typescript-defs + + # Publish + - *step-electron-publish + - move_and_store_all_artifacts + +# List of all jobs. +jobs: + # Layer 0: Docs. Standalone. + ts-compile-doc-change: + executor: + name: linux-docker + size: medium + environment: + <<: *env-linux-medium + <<: *env-testing-build + <<: *steps-electron-ts-compile-for-doc-change + + # Layer 1: Checkout. + linux-checkout-for-workspace: + executor: linux-docker + environment: + <<: *env-linux-2xlarge + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: false + build: false + checkout: true + persist-checkout: true + + linux-make-src-cache: + executor: linux-docker + environment: + <<: *env-linux-2xlarge + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: false + build: false + checkout: true + save-git-cache: true + checkout-to-create-src-cache: true + + linux-checkout-for-native-tests: + executor: linux-docker + environment: + <<: *env-linux-2xlarge + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_pyyaml=True' + steps: + - electron-build: + persist: false + build: false + checkout: true + persist-checkout: true + + linux-checkout-for-native-tests-with-no-patches: + executor: linux-docker + environment: + <<: *env-linux-2xlarge + GCLIENT_EXTRA_ARGS: '--custom-var=apply_patches=False --custom-var=checkout_pyyaml=True' + steps: + - electron-build: + persist: false + build: false + checkout: true + persist-checkout: true + + mac-checkout: + executor: linux-docker + environment: + <<: *env-linux-2xlarge + <<: *env-testing-build + <<: *env-macos-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' + steps: + - electron-build: + persist: false + build: false + checkout: true + persist-checkout: true + restore-src-cache: false + + mac-checkout-for-workspace: + executor: linux-docker + environment: + <<: *env-linux-2xlarge + <<: *env-testing-build + <<: *env-macos-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' + steps: + - electron-build: + persist: false + build: false + checkout: true + persist-checkout: true + + mac-make-src-cache: + executor: linux-docker + environment: + <<: *env-linux-2xlarge + <<: *env-testing-build + <<: *env-macos-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' + steps: + - electron-build: + persist: false + build: false + checkout: true + save-git-cache: true + checkout-to-create-src-cache: true + + # Layer 2: Builds. + linux-x64-testing: + executor: linux-docker + environment: + <<: *env-global + <<: *env-testing-build + <<: *env-ninja-status + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: true + checkout: true + use-out-cache: false + + linux-x64-testing-asan: + executor: linux-docker + environment: + <<: *env-global + <<: *env-testing-build + <<: *env-ninja-status + CHECK_DIST_MANIFEST: '0' + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + GN_EXTRA_ARGS: 'is_asan = true' + steps: + - electron-build: + persist: true + checkout: true + use-out-cache: false + build-nonproprietary-ffmpeg: false + + linux-x64-testing-no-run-as-node: + executor: linux-docker + environment: + <<: *env-linux-2xlarge + <<: *env-testing-build + <<: *env-ninja-status + <<: *env-disable-run-as-node + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: false + checkout: true + use-out-cache: false + + linux-x64-testing-gn-check: + executor: + name: linux-docker + size: medium + environment: + <<: *env-linux-medium + <<: *env-testing-build + <<: *steps-electron-gn-check + + linux-x64-release: + executor: linux-docker + environment: + <<: *env-linux-2xlarge-release + <<: *env-release-build + <<: *env-send-slack-notifications + <<: *env-ninja-status + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: true + checkout: true + + linux-x64-publish: + executor: linux-docker + environment: + <<: *env-linux-2xlarge-release + <<: *env-release-build + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status + steps: + - run: echo running + - when: + condition: + or: + - equal: ["all", << pipeline.parameters.linux-publish-arch-limit >>] + - equal: ["x64", << pipeline.parameters.linux-publish-arch-limit >>] + steps: + - electron-publish: + attach: false + checkout: true + + linux-ia32-testing: + executor: linux-docker + environment: + <<: *env-global + <<: *env-ia32 + <<: *env-testing-build + <<: *env-ninja-status + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: true + checkout: true + use-out-cache: false + + linux-ia32-release: + executor: linux-docker + environment: + <<: *env-linux-2xlarge-release + <<: *env-ia32 + <<: *env-release-build + <<: *env-send-slack-notifications + <<: *env-ninja-status + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: true + checkout: true + + linux-ia32-publish: + executor: linux-docker + environment: + <<: *env-linux-2xlarge-release + <<: *env-ia32 + <<: *env-release-build + <<: *env-32bit-release + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status + steps: + - run: echo running + - when: + condition: + or: + - equal: ["all", << pipeline.parameters.linux-publish-arch-limit >>] + - equal: ["ia32", << pipeline.parameters.linux-publish-arch-limit >>] + steps: + - electron-publish: + attach: false + checkout: true + + linux-arm-testing: + executor: linux-docker + environment: + <<: *env-global + <<: *env-arm + <<: *env-testing-build + <<: *env-ninja-status + TRIGGER_ARM_TEST: true + GENERATE_CROSS_ARCH_SNAPSHOT: true + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: true + checkout: true + use-out-cache: false + + linux-arm-release: + executor: linux-docker + environment: + <<: *env-linux-2xlarge-release + <<: *env-arm + <<: *env-release-build + <<: *env-send-slack-notifications + <<: *env-ninja-status + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: false + checkout: true + + linux-arm-publish: + executor: linux-docker + environment: + <<: *env-linux-2xlarge-release + <<: *env-arm + <<: *env-release-build + <<: *env-32bit-release + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True' + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status + steps: + - run: echo running + - when: + condition: + or: + - equal: ["all", << pipeline.parameters.linux-publish-arch-limit >>] + - equal: ["arm", << pipeline.parameters.linux-publish-arch-limit >>] + steps: + - electron-publish: + attach: false + checkout: true + + linux-arm64-testing: + executor: linux-docker + environment: + <<: *env-global + <<: *env-arm64 + <<: *env-testing-build + <<: *env-ninja-status + TRIGGER_ARM_TEST: true + GENERATE_CROSS_ARCH_SNAPSHOT: true + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: true + checkout: true + use-out-cache: false + + linux-arm64-testing-gn-check: + executor: + name: linux-docker + size: medium + environment: + <<: *env-linux-medium + <<: *env-arm64 + <<: *env-testing-build + <<: *steps-electron-gn-check + + linux-arm64-release: + executor: linux-docker + environment: + <<: *env-linux-2xlarge-release + <<: *env-arm64 + <<: *env-release-build + <<: *env-send-slack-notifications + <<: *env-ninja-status + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: false + checkout: true + + linux-arm64-publish: + executor: linux-docker + environment: + <<: *env-linux-2xlarge-release + <<: *env-arm64 + <<: *env-release-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True' + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status + steps: + - run: echo running + - when: + condition: + or: + - equal: ["all", << pipeline.parameters.linux-publish-arch-limit >>] + - equal: ["arm64", << pipeline.parameters.linux-publish-arch-limit >>] + steps: + - electron-publish: + attach: false + checkout: true + + osx-testing-x64: + executor: macos + environment: + <<: *env-mac-large + <<: *env-testing-build + <<: *env-ninja-status + <<: *env-macos-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' + steps: + - electron-build: + persist: true + checkout: false + checkout-and-assume-cache: true + attach: true + + osx-testing-x64-gn-check: + executor: + name: macos + size: medium + environment: + <<: *env-machine-mac + <<: *env-testing-build + <<: *steps-electron-gn-check + + osx-publish-x64-skip-checkout: + executor: macos + environment: + <<: *env-mac-large-release + <<: *env-release-build + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status + steps: + - run: echo running + - when: + condition: + or: + - equal: ["all", << pipeline.parameters.macos-publish-arch-limit >>] + - equal: ["osx-x64", << pipeline.parameters.macos-publish-arch-limit >>] + steps: + - electron-publish: + attach: true + checkout: false + + osx-publish-arm64-skip-checkout: + executor: macos + environment: + <<: *env-mac-large-release + <<: *env-release-build + <<: *env-apple-silicon + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status + steps: + - run: echo running + - when: + condition: + or: + - equal: ["all", << pipeline.parameters.macos-publish-arch-limit >>] + - equal: ["osx-arm64", << pipeline.parameters.macos-publish-arch-limit >>] + steps: + - electron-publish: + attach: true + checkout: false + + osx-testing-arm64: + executor: macos + environment: + <<: *env-mac-large + <<: *env-testing-build + <<: *env-ninja-status + <<: *env-macos-build + <<: *env-apple-silicon + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' + GENERATE_CROSS_ARCH_SNAPSHOT: true + steps: + - electron-build: + persist: true + checkout: false + checkout-and-assume-cache: true + attach: true + + mas-testing-x64: + executor: macos + environment: + <<: *env-mac-large + <<: *env-mas + <<: *env-testing-build + <<: *env-ninja-status + <<: *env-macos-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' + steps: + - electron-build: + persist: true + checkout: false + checkout-and-assume-cache: true + attach: true + + mas-testing-x64-gn-check: + executor: + name: macos + size: medium + environment: + <<: *env-machine-mac + <<: *env-mas + <<: *env-testing-build + <<: *steps-electron-gn-check + + mas-publish-x64-skip-checkout: + executor: macos + environment: + <<: *env-mac-large-release + <<: *env-mas + <<: *env-release-build + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + steps: + - run: echo running + - when: + condition: + or: + - equal: ["all", << pipeline.parameters.macos-publish-arch-limit >>] + - equal: ["mas-x64", << pipeline.parameters.macos-publish-arch-limit >>] + steps: + - electron-publish: + attach: true + checkout: false + + mas-publish-arm64-skip-checkout: + executor: macos + environment: + <<: *env-mac-large-release + <<: *env-mas-apple-silicon + <<: *env-release-build + UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status + steps: + - run: echo running + - when: + condition: + or: + - equal: ["all", << pipeline.parameters.macos-publish-arch-limit >>] + - equal: ["mas-arm64", << pipeline.parameters.macos-publish-arch-limit >>] + steps: + - electron-publish: + attach: true + checkout: false + + mas-testing-arm64: + executor: macos + environment: + <<: *env-mac-large + <<: *env-testing-build + <<: *env-ninja-status + <<: *env-macos-build + <<: *env-mas-apple-silicon + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' + GENERATE_CROSS_ARCH_SNAPSHOT: true + steps: + - electron-build: + persist: true + checkout: false + checkout-and-assume-cache: true + attach: true + + # Layer 3: Tests. + linux-x64-unittests: + executor: linux-docker + environment: + <<: *env-linux-2xlarge + <<: *env-unittests + <<: *env-headless-testing + <<: *steps-native-tests + + linux-x64-disabled-unittests: + executor: linux-docker + environment: + <<: *env-linux-2xlarge + <<: *env-unittests + <<: *env-headless-testing + TESTS_ARGS: '--only-disabled-tests' + <<: *steps-native-tests + + linux-x64-chromium-unittests: + executor: linux-docker + environment: + <<: *env-linux-2xlarge + <<: *env-unittests + <<: *env-headless-testing + TESTS_ARGS: '--include-disabled-tests' + <<: *steps-native-tests + + linux-x64-browsertests: + executor: linux-docker + environment: + <<: *env-linux-2xlarge + <<: *env-browsertests + <<: *env-testing-build + <<: *env-headless-testing + <<: *steps-native-tests + + linux-x64-testing-tests: + executor: + name: linux-docker + size: medium + environment: + <<: *env-linux-medium + <<: *env-headless-testing + <<: *env-stack-dumping + parallelism: 3 + <<: *steps-tests + + linux-x64-testing-asan-tests: + executor: + name: linux-docker + size: xlarge + environment: + <<: *env-linux-medium + <<: *env-headless-testing + <<: *env-stack-dumping + IS_ASAN: '1' + DISABLE_CRASH_REPORTER_TESTS: '1' + parallelism: 3 + <<: *steps-tests + + linux-x64-testing-nan: + executor: + name: linux-docker + size: medium + environment: + <<: *env-linux-medium + <<: *env-headless-testing + <<: *env-stack-dumping + <<: *steps-test-nan + + linux-x64-testing-node: + executor: linux-docker + environment: + <<: *env-linux-medium + <<: *env-headless-testing + <<: *env-stack-dumping + <<: *steps-test-node + + linux-x64-release-tests: + executor: + name: linux-docker + size: medium + environment: + <<: *env-linux-medium + <<: *env-headless-testing + <<: *env-send-slack-notifications + <<: *steps-tests + + linux-x64-verify-ffmpeg: + executor: + name: linux-docker + size: medium + environment: + <<: *env-linux-medium + <<: *env-headless-testing + <<: *env-send-slack-notifications + <<: *steps-verify-ffmpeg + + linux-ia32-testing-tests: + executor: + name: linux-docker + size: medium + environment: + <<: *env-linux-medium + <<: *env-ia32 + <<: *env-headless-testing + <<: *env-stack-dumping + parallelism: 3 + <<: *steps-tests + + linux-ia32-testing-nan: + executor: + name: linux-docker + size: medium + environment: + <<: *env-linux-medium + <<: *env-ia32 + <<: *env-headless-testing + <<: *env-stack-dumping + <<: *steps-test-nan + + linux-ia32-testing-node: + executor: linux-docker + environment: + <<: *env-linux-medium + <<: *env-ia32 + <<: *env-headless-testing + <<: *env-stack-dumping + <<: *steps-test-node + + linux-ia32-release-tests: + executor: + name: linux-docker + size: medium + environment: + <<: *env-linux-medium + <<: *env-ia32 + <<: *env-headless-testing + <<: *env-send-slack-notifications + <<: *steps-tests + + linux-ia32-verify-ffmpeg: + executor: + name: linux-docker + size: medium + environment: + <<: *env-linux-medium + <<: *env-ia32 + <<: *env-headless-testing + <<: *env-send-slack-notifications + <<: *steps-verify-ffmpeg + + linux-arm-testing-tests: + executor: linux-arm + environment: + <<: *env-arm + <<: *env-global + <<: *env-headless-testing + <<: *env-stack-dumping + <<: *steps-tests + + linux-arm64-testing-tests: + executor: linux-arm64 + environment: + <<: *env-arm64 + <<: *env-global + <<: *env-headless-testing + <<: *env-stack-dumping + <<: *steps-tests + + osx-testing-x64-tests: + executor: + name: macos + size: medium + environment: + <<: *env-mac-large + <<: *env-stack-dumping + parallelism: 2 + <<: *steps-tests + + osx-testing-arm64-tests: + executor: apple-silicon + environment: + <<: *env-mac-large + <<: *env-stack-dumping + <<: *env-apple-silicon + <<: *steps-tests + + mas-testing-x64-tests: + executor: + name: macos + size: medium + environment: + <<: *env-mac-large + <<: *env-stack-dumping + parallelism: 2 + <<: *steps-tests + + mas-testing-arm64-tests: + executor: apple-silicon + environment: + <<: *env-mac-large + <<: *env-stack-dumping + <<: *env-apple-silicon + <<: *steps-tests + + # Layer 4: Summary. + linux-release-summary: + executor: + name: linux-docker + size: medium + environment: + <<: *env-linux-medium + <<: *env-send-slack-notifications + steps: + - *step-maybe-notify-slack-success + +# List all workflows +workflows: + docs-only: + when: + and: + - equal: [false, << pipeline.parameters.run-macos-publish >>] + - equal: [false, << pipeline.parameters.run-linux-publish >>] + - equal: [true, << pipeline.parameters.run-docs-only >>] + jobs: + - ts-compile-doc-change + + publish-linux: + when: << pipeline.parameters.run-linux-publish >> + jobs: + - linux-x64-publish: + context: release-env + - linux-ia32-publish: + context: release-env + - linux-arm-publish: + context: release-env + - linux-arm64-publish: + context: release-env + + publish-macos: + when: << pipeline.parameters.run-macos-publish >> + jobs: + - mac-checkout + - osx-publish-x64-skip-checkout: + requires: + - mac-checkout + context: release-env + - mas-publish-x64-skip-checkout: + requires: + - mac-checkout + context: release-env + - osx-publish-arm64-skip-checkout: + requires: + - mac-checkout + context: release-env + - mas-publish-arm64-skip-checkout: + requires: + - mac-checkout + context: release-env + + build-linux: + when: + and: + - equal: [false, << pipeline.parameters.run-macos-publish >>] + - equal: [false, << pipeline.parameters.run-linux-publish >>] + - equal: [true, << pipeline.parameters.run-build-linux >>] + jobs: + - linux-checkout-for-workspace + - linux-make-src-cache + - linux-x64-testing + - linux-x64-testing-asan + - linux-x64-testing-no-run-as-node + - linux-x64-testing-gn-check: + requires: + - linux-checkout-for-workspace + - linux-x64-testing-tests: + requires: + - linux-x64-testing + - linux-x64-testing-asan-tests: + requires: + - linux-x64-testing-asan + - linux-x64-testing-nan: + requires: + - linux-x64-testing + - linux-x64-testing-node: + requires: + - linux-x64-testing + - linux-ia32-testing + - linux-ia32-testing-tests: + requires: + - linux-ia32-testing + - linux-ia32-testing-nan: + requires: + - linux-ia32-testing + - linux-ia32-testing-node: + requires: + - linux-ia32-testing + - linux-arm-testing + - linux-arm-testing-tests: + filters: + branches: + # Do not run this on forked pull requests + ignore: /pull\/[0-9]+/ + requires: + - linux-arm-testing + - linux-arm64-testing + - linux-arm64-testing-tests: + filters: + branches: + # Do not run this on forked pull requests + ignore: /pull\/[0-9]+/ + requires: + - linux-arm64-testing + - linux-arm64-testing-gn-check: + requires: + - linux-checkout-for-workspace + + build-mac: + when: + and: + - equal: [false, << pipeline.parameters.run-macos-publish >>] + - equal: [false, << pipeline.parameters.run-linux-publish >>] + - equal: [true, << pipeline.parameters.run-build-mac >>] + jobs: + - mac-checkout-for-workspace + - mac-make-src-cache + - osx-testing-x64: + requires: + - mac-make-src-cache + - osx-testing-x64-gn-check: + requires: + - mac-checkout-for-workspace + - osx-testing-x64-tests: + requires: + - osx-testing-x64 + - osx-testing-arm64: + requires: + - mac-make-src-cache + - osx-testing-arm64-tests: + filters: + branches: + # Do not run this on forked pull requests + ignore: /pull\/[0-9]+/ + requires: + - osx-testing-arm64 + - mas-testing-x64: + requires: + - mac-make-src-cache + - mas-testing-x64-gn-check: + requires: + - mac-checkout-for-workspace + - mas-testing-x64-tests: + requires: + - mas-testing-x64 + - mas-testing-arm64: + requires: + - mac-make-src-cache + - mas-testing-arm64-tests: + filters: + branches: + # Do not run this on forked pull requests + ignore: /pull\/[0-9]+/ + requires: + - mas-testing-arm64 diff --git a/.circleci/config.yml b/.circleci/config.yml index 5afe973dd5..344b434438 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,21 +1,29 @@ version: 2.1 +# Required for dynamic configuration +setup: true + +# Orbs +orbs: + path-filtering: circleci/path-filtering@0.1.0 + +# All input parameters to pass to build config parameters: + run-docs-only: + type: boolean + default: false + upload-to-s3: type: string default: '1' - run-lint: - type: boolean - default: true - run-build-linux: type: boolean - default: true + default: false run-build-mac: type: boolean - default: true + default: false run-linux-publish: type: boolean @@ -35,6 +43,14 @@ parameters: default: all enum: ["all", "osx-x64", "osx-arm64", "mas-x64", "mas-arm64"] +# Envs +env-global: &env-global + ELECTRON_OUT_DIR: Default + +env-linux-medium: &env-linux-medium + <<: *env-global + NUMBER_OF_NINJA_PROCESSES: 3 + # Executors executors: linux-docker: @@ -48,987 +64,11 @@ executors: - image: ghcr.io/electron/build:27db4a3e3512bfd2e47f58cea69922da0835f1d9 resource_class: << parameters.size >> - macos: - parameters: - size: - description: "macOS executor size" - default: large - type: enum - enum: ["medium", "large"] - macos: - xcode: "12.4.0" - resource_class: << parameters.size >> - - # Electron Runners - apple-silicon: - resource_class: electronjs/macos-arm64 - machine: true - - linux-arm: - resource_class: electronjs/linux-arm - machine: true - - linux-arm64: - resource_class: electronjs/linux-arm64 - machine: true - -# The config expects the following environment variables to be set: -# - "SLACK_WEBHOOK" Slack hook URL to send notifications. -# -# The publishing scripts expect access tokens to be defined as env vars, -# but those are not covered here. -# -# CircleCI docs on variables: -# https://circleci.com/docs/2.0/env-vars/ - -# Build configurations options. -env-testing-build: &env-testing-build - GN_CONFIG: //electron/build/args/testing.gn - CHECK_DIST_MANIFEST: '1' - -env-release-build: &env-release-build - GN_CONFIG: //electron/build/args/release.gn - STRIP_BINARIES: true - GENERATE_SYMBOLS: true - CHECK_DIST_MANIFEST: '1' - IS_RELEASE: true - -env-headless-testing: &env-headless-testing - DISPLAY: ':99.0' - -env-stack-dumping: &env-stack-dumping - ELECTRON_ENABLE_STACK_DUMPING: '1' - -env-browsertests: &env-browsertests - GN_CONFIG: //electron/build/args/native_tests.gn - BUILD_TARGET: electron/spec:chromium_browsertests - TESTS_CONFIG: src/electron/spec/configs/browsertests.yml - -env-unittests: &env-unittests - GN_CONFIG: //electron/build/args/native_tests.gn - BUILD_TARGET: electron/spec:chromium_unittests - TESTS_CONFIG: src/electron/spec/configs/unittests.yml - -# Build targets options. -env-ia32: &env-ia32 - GN_EXTRA_ARGS: 'target_cpu = "x86"' - NPM_CONFIG_ARCH: ia32 - TARGET_ARCH: ia32 - -env-arm: &env-arm - GN_EXTRA_ARGS: 'target_cpu = "arm"' - MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm - BUILD_NATIVE_MKSNAPSHOT: 1 - TARGET_ARCH: arm - -env-apple-silicon: &env-apple-silicon - GN_EXTRA_ARGS: 'target_cpu = "arm64" use_prebuilt_v8_context_snapshot = true' - TARGET_ARCH: arm64 - USE_PREBUILT_V8_CONTEXT_SNAPSHOT: 1 - npm_config_arch: arm64 - -env-arm64: &env-arm64 - GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false' - MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm64 - BUILD_NATIVE_MKSNAPSHOT: 1 - TARGET_ARCH: arm64 - -env-mas: &env-mas - GN_EXTRA_ARGS: 'is_mas_build = true' - MAS_BUILD: 'true' - -env-mas-apple-silicon: &env-mas-apple-silicon - GN_EXTRA_ARGS: 'target_cpu = "arm64" is_mas_build = true use_prebuilt_v8_context_snapshot = true' - MAS_BUILD: 'true' - TARGET_ARCH: arm64 - USE_PREBUILT_V8_CONTEXT_SNAPSHOT: 1 - -env-send-slack-notifications: &env-send-slack-notifications - NOTIFY_SLACK: true - -env-global: &env-global - ELECTRON_OUT_DIR: Default - -env-linux-medium: &env-linux-medium - <<: *env-global - NUMBER_OF_NINJA_PROCESSES: 3 - -env-linux-2xlarge: &env-linux-2xlarge - <<: *env-global - NUMBER_OF_NINJA_PROCESSES: 34 - -env-linux-2xlarge-release: &env-linux-2xlarge-release - <<: *env-global - NUMBER_OF_NINJA_PROCESSES: 16 - -env-machine-mac: &env-machine-mac - <<: *env-global - NUMBER_OF_NINJA_PROCESSES: 6 - -env-mac-large: &env-mac-large - <<: *env-global - NUMBER_OF_NINJA_PROCESSES: 18 - -env-mac-large-release: &env-mac-large-release - <<: *env-global - NUMBER_OF_NINJA_PROCESSES: 8 - -env-ninja-status: &env-ninja-status - NINJA_STATUS: "[%r processes, %f/%t @ %o/s : %es] " - -env-disable-run-as-node: &env-disable-run-as-node - GN_BUILDFLAG_ARGS: 'enable_run_as_node = false' - -env-32bit-release: &env-32bit-release - # Set symbol level to 1 for 32 bit releases because of https://crbug.com/648948 - GN_BUILDFLAG_ARGS: 'symbol_level = 1' - -env-macos-build: &env-macos-build - # Disable pre-compiled headers to reduce out size, only useful for rebuilds - GN_BUILDFLAG_ARGS: 'enable_precompiled_headers = false' - -# Individual (shared) steps. -step-maybe-notify-slack-failure: &step-maybe-notify-slack-failure - run: - name: Send a Slack notification on failure - command: | - if [ "$NOTIFY_SLACK" == "true" ]; then - export MESSAGE="Build failed for *<$CIRCLE_BUILD_URL|$CIRCLE_JOB>* nightly build from *$CIRCLE_BRANCH*." - curl -g -H "Content-Type: application/json" -X POST \ - -d "{\"text\": \"$MESSAGE\", \"attachments\": [{\"color\": \"#FC5C3C\",\"title\": \"$CIRCLE_JOB nightly build results\",\"title_link\": \"$CIRCLE_BUILD_URL\"}]}" $SLACK_WEBHOOK - fi - when: on_fail - -step-maybe-notify-slack-success: &step-maybe-notify-slack-success - run: - name: Send a Slack notification on success - command: | - if [ "$NOTIFY_SLACK" == "true" ]; then - export MESSAGE="Build succeeded for *<$CIRCLE_BUILD_URL|$CIRCLE_JOB>* nightly build from *$CIRCLE_BRANCH*." - curl -g -H "Content-Type: application/json" -X POST \ - -d "{\"text\": \"$MESSAGE\", \"attachments\": [{\"color\": \"good\",\"title\": \"$CIRCLE_JOB nightly build results\",\"title_link\": \"$CIRCLE_BUILD_URL\"}]}" $SLACK_WEBHOOK - fi - when: on_success - -step-maybe-cleanup-arm64-mac: &step-maybe-cleanup-arm64-mac - run: - name: Cleanup after testing - command: | - if [ "$TARGET_ARCH" == "arm64" ] &&[ "`uname`" == "Darwin" ]; then - killall Electron || echo "No Electron processes left running" - killall Safari || echo "No Safari processes left running" - rm -rf ~/Library/Application\ Support/Electron* - rm -rf ~/Library/Application\ Support/electron* - security delete-generic-password -l "Chromium Safe Storage" || echo "✓ Keychain does not contain password from tests" - security delete-generic-password -l "Electron Test Main Safe Storage" || echo "✓ Keychain does not contain password from tests" - elif [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then - XVFB=/usr/bin/Xvfb - /sbin/start-stop-daemon --stop --exec $XVFB || echo "Xvfb not running" - pkill electron || echo "electron not running" - rm -rf ~/.config/Electron* - rm -rf ~/.config/electron* - fi - - when: always - +# List of always run steps step-checkout-electron: &step-checkout-electron checkout: path: src/electron -step-depot-tools-get: &step-depot-tools-get - run: - name: Get depot tools - command: | - git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git - -step-depot-tools-add-to-path: &step-depot-tools-add-to-path - run: - name: Add depot tools to PATH - command: echo 'export PATH="$PATH:'"$PWD"'/depot_tools"' >> $BASH_ENV - -step-gclient-sync: &step-gclient-sync - run: - name: Gclient sync - command: | - # If we did not restore a complete sync then we need to sync for realz - if [ ! -s "src/electron/.circle-sync-done" ]; then - gclient config \ - --name "src/electron" \ - --unmanaged \ - $GCLIENT_EXTRA_ARGS \ - "$CIRCLE_REPOSITORY_URL" - - ELECTRON_USE_THREE_WAY_MERGE_FOR_PATCHES=1 gclient sync --with_branch_heads --with_tags - if [ "$IS_RELEASE" != "true" ]; then - # Re-export all the patches to check if there were changes. - python src/electron/script/export_all_patches.py src/electron/patches/config.json - cd src/electron - git update-index --refresh || true - if ! git diff-index --quiet HEAD --; then - # There are changes to the patches. Make a git commit with the updated patches - git add patches - GIT_COMMITTER_NAME="PatchUp" GIT_COMMITTER_EMAIL="73610968+patchup[bot]@users.noreply.github.com" git commit -m "chore: update patches" --author="PatchUp <73610968+patchup[bot]@users.noreply.github.com>" - # Export it - mkdir -p ../../patches - git format-patch -1 --stdout --keep-subject --no-stat --full-index > ../../patches/update-patches.patch - if (node ./script/push-patch.js 2> /dev/null > /dev/null); then - echo - echo "======================================================================" - echo "Changes to the patches when applying, we have auto-pushed the diff to the current branch" - echo "A new CI job will kick off shortly" - echo "======================================================================" - exit 1 - else - echo - echo "======================================================================" - echo "There were changes to the patches when applying." - echo "Check the CI artifacts for a patch you can apply to fix it." - echo "======================================================================" - exit 1 - fi - fi - fi - fi - -step-setup-env-for-build: &step-setup-env-for-build - run: - name: Setup Environment Variables - command: | - # To find `gn` executable. - echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV - -step-setup-goma-for-build: &step-setup-goma-for-build - run: - name: Setup Goma - command: | - echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV - if [ "`uname`" == "Darwin" ]; then - echo 'ulimit -n 10000' >> $BASH_ENV - echo 'sudo launchctl limit maxfiles 65536 200000' >> $BASH_ENV - fi - if [ ! -z "$RAW_GOMA_AUTH" ]; then - echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config - fi - git clone https://github.com/electron/build-tools.git - cd build-tools - npm install - mkdir third_party - node -e "require('./src/utils/goma.js').downloadAndPrepare({ gomaOneForAll: true })" - export GOMA_FALLBACK_ON_AUTH_FAILURE=true - third_party/goma/goma_ctl.py ensure_start - echo 'export GN_GOMA_FILE='`node -e "console.log(require('./src/utils/goma.js').gnFilePath)"` >> $BASH_ENV - echo 'export LOCAL_GOMA_DIR='`node -e "console.log(require('./src/utils/goma.js').dir)"` >> $BASH_ENV - echo 'export GOMA_FALLBACK_ON_AUTH_FAILURE=true' >> $BASH_ENV - cd .. - -step-restore-brew-cache: &step-restore-brew-cache - restore_cache: - paths: - - /usr/local/Cellar/gnu-tar - - /usr/local/bin/gtar - keys: - - v4-brew-cache-{{ arch }} - -step-save-brew-cache: &step-save-brew-cache - save_cache: - paths: - - /usr/local/Cellar/gnu-tar - - /usr/local/bin/gtar - key: v4-brew-cache-{{ arch }} - name: Persisting brew cache - -step-get-more-space-on-mac: &step-get-more-space-on-mac - run: - name: Free up space on MacOS - command: | - if [ "`uname`" == "Darwin" ]; then - sudo mkdir -p $TMPDIR/del-target - if [ "$TARGET_ARCH" == "arm64" ]; then - # Remount the root volume as writable, don't ask questions plz - sudo mount -uw / - fi - tmpify() { - if [ -d "$1" ]; then - sudo mv "$1" $TMPDIR/del-target/$(echo $1|shasum -a 256|head -n1|cut -d " " -f1) - fi - } - - strip_arm_deep() { - opwd=$(pwd) - cd $1 - f=$(find . -perm +111 -type f) - for fp in $f - do - if [[ $(file "$fp") == *"universal binary"* ]]; then - if [[ $(file "$fp") == *"arm64e)"* ]]; then - sudo lipo -remove arm64e "$fp" -o "$fp" || true - fi - if [[ $(file "$fp") == *"arm64)"* ]]; then - sudo lipo -remove arm64 "$fp" -o "$fp" || true - fi - fi - done - - cd $opwd - } - - tmpify /Library/Developer/CoreSimulator - tmpify ~/Library/Developer/CoreSimulator - tmpify $(xcode-select -p)/Platforms/AppleTVOS.platform - tmpify $(xcode-select -p)/Platforms/iPhoneOS.platform - tmpify $(xcode-select -p)/Platforms/WatchOS.platform - tmpify $(xcode-select -p)/Platforms/WatchSimulator.platform - tmpify $(xcode-select -p)/Platforms/AppleTVSimulator.platform - tmpify $(xcode-select -p)/Platforms/iPhoneSimulator.platform - tmpify $(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/metal/ios - tmpify $(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift - tmpify $(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.0 - tmpify ~/.rubies - tmpify ~/Library/Caches/Homebrew - tmpify /usr/local/Homebrew - sudo rm -rf $TMPDIR/del-target - - if [ "$TARGET_ARCH" == "arm64" ]; then - sudo rm -rf "/System/Library/Desktop Pictures" - sudo rm -rf /System/Library/Templates/Data - sudo rm -rf /System/Library/Speech/Voices - sudo rm -rf "/System/Library/Screen Savers" - sudo rm -rf /System/Volumes/Data/Library/Developer/CommandLineTools/SDKs - sudo rm -rf "/System/Volumes/Data/Library/Application Support/Apple/Photos/Print Products" - sudo rm -rf /System/Volumes/Data/Library/Java - sudo rm -rf /System/Volumes/Data/Library/Ruby - sudo rm -rf /System/Volumes/Data/Library/Printers - sudo rm -rf /System/iOSSupport - sudo rm -rf /System/Applications/*.app - sudo rm -rf /System/Applications/Utilities/*.app - sudo rm -rf /System/Library/LinguisticData - sudo rm -rf /System/Volumes/Data/private/var/db/dyld/* - # sudo rm -rf /System/Library/Fonts/* - # sudo rm -rf /System/Library/PreferencePanes - sudo rm -rf /System/Library/AssetsV2/* - sudo rm -rf /Applications/Safari.app - sudo rm -rf ~/project/src/build/linux - sudo rm -rf ~/project/src/third_party/catapult/tracing/test_data - sudo rm -rf ~/project/src/third_party/angle/third_party/VK-GL-CTS - - # lipo off some huge binaries arm64 versions to save space - strip_arm_deep $(xcode-select -p)/../SharedFrameworks - strip_arm_deep /System/Volumes/Data/Library/Developer/CommandLineTools/usr - fi - fi - background: true - -# On macOS delete all .git directories under src/ expect for -# third_party/angle/ because of build time generation of file -# gen/angle/commit.h depends on third_party/angle/.git/HEAD -# https://chromium-review.googlesource.com/c/angle/angle/+/2074924 -# TODO: maybe better to always leave out */.git/HEAD file for all targets ? -step-delete-git-directories: &step-delete-git-directories - run: - name: Delete all .git directories under src on MacOS to free space - command: | - if [ "`uname`" == "Darwin" ]; then - cd src - ( find . -type d -name ".git" -not -path "./third_party/angle/*" ) | xargs rm -rf - fi - -# On macOS the yarn install command during gclient sync was run on a linux -# machine and therefore installed a slightly different set of dependencies -# Notably "fsevents" is a macOS only dependency, we rerun yarn install once -# we are on a macOS machine to get the correct state -step-install-npm-deps-on-mac: &step-install-npm-deps-on-mac - run: - name: Install node_modules on MacOS - command: | - if [ "`uname`" == "Darwin" ]; then - cd src/electron - node script/yarn install - fi - -# This step handles the differences between the linux "gclient sync" -# and the expected state on macOS -step-fix-sync: &step-fix-sync - run: - name: Fix Sync - command: | - if [ "`uname`" == "Darwin" ]; then - # Fix Clang Install (wrong binary) - rm -rf src/third_party/llvm-build - python3 src/tools/clang/scripts/update.py - fi - - cd src/third_party/angle - git remote set-url origin https://chromium.googlesource.com/angle/angle.git - git fetch - -step-install-signing-cert-on-mac: &step-install-signing-cert-on-mac - run: - name: Import and trust self-signed codesigning cert on MacOS - command: | - if [ "$TARGET_ARCH" != "arm64" ] && [ "`uname`" == "Darwin" ]; then - cd src/electron - ./script/codesign/generate-identity.sh - fi - -step-install-gnutar-on-mac: &step-install-gnutar-on-mac - run: - name: Install gnu-tar on macos - command: | - if [ "`uname`" == "Darwin" ]; then - if [ ! -d /usr/local/Cellar/gnu-tar/ ]; then - brew update - brew install gnu-tar - fi - ln -fs /usr/local/bin/gtar /usr/local/bin/tar - fi - -step-gn-gen-default: &step-gn-gen-default - run: - name: Default GN gen - command: | - cd src - gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" - -step-gn-check: &step-gn-check - run: - name: GN check - command: | - cd src - gn check out/Default //electron:electron_lib - gn check out/Default //electron:electron_app - gn check out/Default //electron/shell/common/api:mojo - # Check the hunspell filenames - node electron/script/gen-hunspell-filenames.js --check - node electron/script/gen-libc++-filenames.js --check - -step-electron-build: &step-electron-build - run: - name: Electron build - no_output_timeout: 30m - command: | - # On arm platforms we generate a cross-arch ffmpeg that ninja does not seem - # to realize is not correct / should be rebuilt. We delete it here so it is - # rebuilt - if [ "$TRIGGER_ARM_TEST" == "true" ]; then - rm -f src/out/Default/libffmpeg.so - fi - cd src - # Enable if things get really bad - # if [ "$TARGET_ARCH" == "arm64" ] &&[ "`uname`" == "Darwin" ]; then - # diskutil erasevolume HFS+ "xcode_disk" `hdiutil attach -nomount ram://12582912` - # mv /Applications/Xcode-12.beta.5.app /Volumes/xcode_disk/ - # ln -s /Volumes/xcode_disk/Xcode-12.beta.5.app /Applications/Xcode-12.beta.5.app - # fi - - # Lets generate a snapshot and mksnapshot and then delete all the x-compiled generated files to save space - if [ "$USE_PREBUILT_V8_CONTEXT_SNAPSHOT" == "1" ]; then - ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES - ninja -C out/Default tools/v8_context_snapshot -j $NUMBER_OF_NINJA_PROCESSES - gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args - (cd out/Default; zip mksnapshot.zip mksnapshot_args clang_x64_v8_arm64/gen/v8/embedded.S) - rm -rf out/Default/clang_x64_v8_arm64/gen - rm -rf out/Default/clang_x64_v8_arm64/obj - rm -rf out/Default/clang_x64/obj - - # Regenerate because we just deleted some ninja files - gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" - fi - NINJA_SUMMARIZE_BUILD=1 autoninja -C out/Default electron -j $NUMBER_OF_NINJA_PROCESSES - cp out/Default/.ninja_log out/electron_ninja_log - node electron/script/check-symlinks.js - -step-native-unittests-build: &step-native-unittests-build - run: - name: Build native test targets - no_output_timeout: 30m - command: | - cd src - ninja -C out/Default shell_browser_ui_unittests -j $NUMBER_OF_NINJA_PROCESSES - -step-maybe-electron-dist-strip: &step-maybe-electron-dist-strip - run: - name: Strip electron binaries - command: | - if [ "$STRIP_BINARIES" == "true" ] && [ "`uname`" == "Linux" ]; then - if [ x"$TARGET_ARCH" == x ]; then - target_cpu=x64 - elif [ "$TARGET_ARCH" == "ia32" ]; then - target_cpu=x86 - else - target_cpu="$TARGET_ARCH" - fi - cd src - electron/script/copy-debug-symbols.py --target-cpu="$target_cpu" --out-dir=out/Default/debug --compress - electron/script/strip-binaries.py --target-cpu="$target_cpu" - electron/script/add-debug-link.py --target-cpu="$target_cpu" --debug-dir=out/Default/debug - fi - -step-electron-dist-build: &step-electron-dist-build - run: - name: Build dist.zip - command: | - cd src - if [ "$SKIP_DIST_ZIP" != "1" ]; then - ninja -C out/Default electron:electron_dist_zip - if [ "$CHECK_DIST_MANIFEST" == "1" ]; then - if [ "`uname`" == "Darwin" ]; then - target_os=mac - target_cpu=x64 - if [ x"$MAS_BUILD" == x"true" ]; then - target_os=mac_mas - fi - if [ "$TARGET_ARCH" == "arm64" ]; then - target_cpu=arm64 - fi - elif [ "`uname`" == "Linux" ]; then - target_os=linux - if [ x"$TARGET_ARCH" == x ]; then - target_cpu=x64 - elif [ "$TARGET_ARCH" == "ia32" ]; then - target_cpu=x86 - else - target_cpu="$TARGET_ARCH" - fi - else - echo "Unknown system: `uname`" - exit 1 - fi - electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.$target_cpu.manifest - fi - fi - -step-electron-chromedriver-build: &step-electron-chromedriver-build - run: - name: Build chromedriver.zip - command: | - cd src - if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then - gn gen out/chromedriver --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") is_component_ffmpeg=false proprietary_codecs=false $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" - export CHROMEDRIVER_DIR="out/chromedriver" - else - export CHROMEDRIVER_DIR="out/Default" - fi - ninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver -j $NUMBER_OF_NINJA_PROCESSES - if [ "`uname`" == "Linux" ]; then - electron/script/strip-binaries.py --target-cpu="$TARGET_ARCH" --file $PWD/$CHROMEDRIVER_DIR/chromedriver - fi - ninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver_zip - if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then - cp out/chromedriver/chromedriver.zip out/Default - fi - -step-nodejs-headers-build: &step-nodejs-headers-build - run: - name: Build Node.js headers - command: | - cd src - ninja -C out/Default third_party/electron_node:headers - -step-electron-publish: &step-electron-publish - run: - name: Publish Electron Dist - command: | - if [ "`uname`" == "Darwin" ]; then - rm -rf src/out/Default/obj - fi - - cd src/electron - if [ "$UPLOAD_TO_S3" == "1" ]; then - echo 'Uploading Electron release distribution to S3' - script/release/uploaders/upload.py --verbose --upload_to_s3 - else - echo 'Uploading Electron release distribution to Github releases' - script/release/uploaders/upload.py --verbose - fi - -step-persist-data-for-tests: &step-persist-data-for-tests - persist_to_workspace: - root: . - paths: - # Build artifacts - - src/out/Default/dist.zip - - src/out/Default/mksnapshot.zip - - src/out/Default/chromedriver.zip - - src/out/Default/shell_browser_ui_unittests - - src/out/Default/gen/node_headers - - src/out/ffmpeg/ffmpeg.zip - - src/electron - - src/third_party/electron_node - - src/third_party/nan - - src/cross-arch-snapshots - - src/third_party/llvm-build - - src/build/linux - - src/buildtools/third_party/libc++ - - src/buildtools/third_party/libc++abi - - src/out/Default/obj/buildtools/third_party - -step-electron-dist-unzip: &step-electron-dist-unzip - run: - name: Unzip dist.zip - command: | - cd src/out/Default - # -o overwrite files WITHOUT prompting - # TODO(alexeykuzmin): Remove '-o' when it's no longer needed. - # -: allows to extract archive members into locations outside - # of the current ``extraction root folder''. - # ASan builds have the llvm-symbolizer binaries listed as - # runtime_deps, with their paths as `../../third_party/...` - # unzip exits with non-zero code on such zip files unless -: is - # passed. - unzip -:o dist.zip - -step-ffmpeg-unzip: &step-ffmpeg-unzip - run: - name: Unzip ffmpeg.zip - command: | - cd src/out/ffmpeg - unzip -:o ffmpeg.zip - -step-mksnapshot-unzip: &step-mksnapshot-unzip - run: - name: Unzip mksnapshot.zip - command: | - cd src/out/Default - unzip -:o mksnapshot.zip - -step-chromedriver-unzip: &step-chromedriver-unzip - run: - name: Unzip chromedriver.zip - command: | - cd src/out/Default - unzip -:o chromedriver.zip - -step-ffmpeg-gn-gen: &step-ffmpeg-gn-gen - run: - name: ffmpeg GN gen - command: | - cd src - gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS" - -step-ffmpeg-build: &step-ffmpeg-build - run: - name: Non proprietary ffmpeg build - command: | - cd src - ninja -C out/ffmpeg electron:electron_ffmpeg_zip -j $NUMBER_OF_NINJA_PROCESSES - -step-verify-ffmpeg: &step-verify-ffmpeg - run: - name: Verify ffmpeg - command: | - cd src - python electron/script/verify-ffmpeg.py --source-root "$PWD" --build-dir out/Default --ffmpeg-path out/ffmpeg - -step-verify-mksnapshot: &step-verify-mksnapshot - run: - name: Verify mksnapshot - command: | - if [ "$IS_ASAN" != "1" ]; then - cd src - if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then - python electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default --snapshot-files-dir $PWD/cross-arch-snapshots - else - python electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default - fi - fi - -step-verify-chromedriver: &step-verify-chromedriver - run: - name: Verify ChromeDriver - command: | - if [ "$IS_ASAN" != "1" ]; then - cd src - python electron/script/verify-chromedriver.py --source-root "$PWD" --build-dir out/Default - fi - -step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing - run: - name: Setup for headless testing - command: | - if [ "`uname`" != "Darwin" ]; then - sh -e /etc/init.d/xvfb start - fi - -step-show-goma-stats: &step-show-goma-stats - run: - shell: /bin/bash - name: Check goma stats after build - command: | - set +e - set +o pipefail - $LOCAL_GOMA_DIR/goma_ctl.py stat - $LOCAL_GOMA_DIR/diagnose_goma_log.py - true - when: always - -step-mksnapshot-build: &step-mksnapshot-build - run: - name: mksnapshot build - command: | - cd src - if [ "$USE_PREBUILT_V8_CONTEXT_SNAPSHOT" != "1" ]; then - ninja -C out/Default electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES - gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args - fi - if [ "`uname`" != "Darwin" ]; then - if [ "$TARGET_ARCH" == "arm" ]; then - electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/mksnapshot - electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/v8_context_snapshot_generator - elif [ "$TARGET_ARCH" == "arm64" ]; then - electron/script/strip-binaries.py --file $PWD/out/Default/clang_x64_v8_arm64/mksnapshot - electron/script/strip-binaries.py --file $PWD/out/Default/clang_x64_v8_arm64/v8_context_snapshot_generator - else - electron/script/strip-binaries.py --file $PWD/out/Default/mksnapshot - electron/script/strip-binaries.py --file $PWD/out/Default/v8_context_snapshot_generator - fi - fi - if [ "$USE_PREBUILT_V8_CONTEXT_SNAPSHOT" != "1" ] && [ "$SKIP_DIST_ZIP" != "1" ]; then - ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES - (cd out/Default; zip mksnapshot.zip mksnapshot_args gen/v8/embedded.S) - fi - -step-hunspell-build: &step-hunspell-build - run: - name: hunspell build - command: | - cd src - if [ "$SKIP_DIST_ZIP" != "1" ]; then - ninja -C out/Default electron:hunspell_dictionaries_zip -j $NUMBER_OF_NINJA_PROCESSES - fi - -step-maybe-generate-libcxx: &step-maybe-generate-libcxx - run: - name: maybe generate libcxx - command: | - cd src - if [ "`uname`" == "Linux" ]; then - ninja -C out/Default electron:libcxx_headers_zip -j $NUMBER_OF_NINJA_PROCESSES - ninja -C out/Default electron:libcxxabi_headers_zip -j $NUMBER_OF_NINJA_PROCESSES - ninja -C out/Default electron:libcxx_objects_zip -j $NUMBER_OF_NINJA_PROCESSES - fi - -step-maybe-generate-breakpad-symbols: &step-maybe-generate-breakpad-symbols - run: - name: Generate breakpad symbols - no_output_timeout: 30m - command: | - if [ "$GENERATE_SYMBOLS" == "true" ]; then - cd src - ninja -C out/Default electron:electron_symbols - fi - -step-maybe-zip-symbols: &step-maybe-zip-symbols - run: - name: Zip symbols - command: | - cd src - export BUILD_PATH="$PWD/out/Default" - ninja -C out/Default electron:licenses - ninja -C out/Default electron:electron_version - DELETE_DSYMS_AFTER_ZIP=1 electron/script/zip-symbols.py -b $BUILD_PATH - -step-maybe-cross-arch-snapshot: &step-maybe-cross-arch-snapshot - run: - name: Generate cross arch snapshot (arm/arm64) - command: | - if [ "$GENERATE_CROSS_ARCH_SNAPSHOT" == "true" ] && [ -z "$CIRCLE_PR_NUMBER" ]; then - cd src - if [ "$TARGET_ARCH" == "arm" ]; then - export MKSNAPSHOT_PATH="clang_x86_v8_arm" - elif [ "$TARGET_ARCH" == "arm64" ]; then - export MKSNAPSHOT_PATH="clang_x64_v8_arm64" - fi - cp "out/Default/$MKSNAPSHOT_PATH/mksnapshot" out/Default - cp "out/Default/$MKSNAPSHOT_PATH/v8_context_snapshot_generator" out/Default - if [ "`uname`" == "Linux" ]; then - cp "out/Default/$MKSNAPSHOT_PATH/libffmpeg.so" out/Default - elif [ "`uname`" == "Darwin" ]; then - cp "out/Default/$MKSNAPSHOT_PATH/libffmpeg.dylib" out/Default - fi - python electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default --create-snapshot-only - mkdir cross-arch-snapshots - cp out/Default-mksnapshot-test/*.bin cross-arch-snapshots - fi - -step-maybe-generate-typescript-defs: &step-maybe-generate-typescript-defs - run: - name: Generate type declarations - command: | - if [ "`uname`" == "Darwin" ]; then - cd src/electron - node script/yarn create-typescript-definitions - fi - -step-fix-known-hosts-linux: &step-fix-known-hosts-linux - run: - name: Fix Known Hosts on Linux - command: | - if [ "`uname`" == "Linux" ]; then - ./src/electron/.circleci/fix-known-hosts.sh - fi - -# Checkout Steps -step-generate-deps-hash: &step-generate-deps-hash - run: - name: Generate DEPS Hash - command: node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target - -step-touch-sync-done: &step-touch-sync-done - run: - name: Touch Sync Done - command: touch src/electron/.circle-sync-done - -# Restore exact src cache based on the hash of DEPS and patches/* -# If no cache is matched EXACTLY then the .circle-sync-done file is empty -# If a cache is matched EXACTLY then the .circle-sync-done file contains "done" -step-maybe-restore-src-cache: &step-maybe-restore-src-cache - restore_cache: - keys: - - v8-src-cache-{{ checksum "src/electron/.depshash" }} - name: Restoring src cache -step-maybe-restore-src-cache-marker: &step-maybe-restore-src-cache-marker - restore_cache: - keys: - - v1-src-cache-marker-{{ checksum "src/electron/.depshash" }} - name: Restoring src cache marker - -# Restore exact or closest git cache based on the hash of DEPS and .circle-sync-done -# If the src cache was restored above then this will match an empty cache -# If the src cache was not restored above then this will match a close git cache -step-maybe-restore-git-cache: &step-maybe-restore-git-cache - restore_cache: - paths: - - ~/.gclient-cache - keys: - - v2-gclient-cache-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }} - - v2-gclient-cache-{{ checksum "src/electron/.circle-sync-done" }} - name: Conditionally restoring git cache - -step-restore-out-cache: &step-restore-out-cache - restore_cache: - paths: - - ./src/out/Default - keys: - - v9-out-cache-{{ checksum "src/electron/.depshash" }}-{{ checksum "src/electron/.depshash-target" }} - name: Restoring out cache - -step-set-git-cache-path: &step-set-git-cache-path - run: - name: Set GIT_CACHE_PATH to make gclient to use the cache - command: | - # CircleCI does not support interpolation when setting environment variables. - # https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command - echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV - -# Persist the git cache based on the hash of DEPS and .circle-sync-done -# If the src cache was restored above then this will persist an empty cache -step-save-git-cache: &step-save-git-cache - save_cache: - paths: - - ~/.gclient-cache - key: v2-gclient-cache-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }} - name: Persisting git cache - -step-save-out-cache: &step-save-out-cache - save_cache: - paths: - - ./src/out/Default - key: v9-out-cache-{{ checksum "src/electron/.depshash" }}-{{ checksum "src/electron/.depshash-target" }} - name: Persisting out cache - -step-run-electron-only-hooks: &step-run-electron-only-hooks - run: - name: Run Electron Only Hooks - command: gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]" - -step-generate-deps-hash-cleanly: &step-generate-deps-hash-cleanly - run: - name: Generate DEPS Hash - command: (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target - -# Mark the sync as done for future cache saving -step-mark-sync-done: &step-mark-sync-done - run: - name: Mark Sync Done - command: echo DONE > src/electron/.circle-sync-done - -# Minimize the size of the cache -step-minimize-workspace-size-from-checkout: &step-minimize-workspace-size-from-checkout - run: - name: Remove some unused data to avoid storing it in the workspace/cache - command: | - rm -rf src/android_webview - rm -rf src/ios/chrome - rm -rf src/third_party/blink/web_tests - rm -rf src/third_party/blink/perf_tests - rm -rf src/third_party/WebKit/LayoutTests - rm -rf third_party/electron_node/deps/openssl - rm -rf third_party/electron_node/deps/v8 - rm -rf chrome/test/data/xr/webvr_info - -# Save the src cache based on the deps hash -step-save-src-cache: &step-save-src-cache - save_cache: - paths: - - /var/portal - key: v8-src-cache-{{ checksum "/var/portal/src/electron/.depshash" }} - name: Persisting src cache -step-make-src-cache-marker: &step-make-src-cache-marker - run: - name: Making src cache marker - command: touch .src-cache-marker -step-save-src-cache-marker: &step-save-src-cache-marker - save_cache: - paths: - - .src-cache-marker - key: v1-src-cache-marker-{{ checksum "/var/portal/src/electron/.depshash" }} - -# Check for doc only change -step-check-for-doc-only-change: &step-check-for-doc-only-change - run: - name: Check if commit is doc only change - command: | - cd src/electron - node script/yarn install --frozen-lockfile - if node script/doc-only-change.js --prNumber=$CIRCLE_PR_NUMBER --prURL=$CIRCLE_PULL_REQUEST --prBranch=$CIRCLE_BRANCH; then - #PR is doc only change; save file with value true to indicate doc only change - echo "true" > .skip-ci-build - else - #PR is not a doc only change; create empty file to indicate check has been done - touch .skip-ci-build - fi - -step-persist-doc-only-change: &step-persist-doc-only-change - persist_to_workspace: - root: . - paths: - - src/electron/.skip-ci-build - -step-maybe-early-exit-doc-only-change: &step-maybe-early-exit-doc-only-change - run: - name: Shortcircuit build if doc only change - command: | - if [ -s src/electron/.skip-ci-build ]; then - circleci-agent step halt - fi - -step-maybe-early-exit-no-doc-change: &step-maybe-early-exit-no-doc-change - run: - name: Shortcircuit job if change is not doc only - command: | - if [ ! -s src/electron/.skip-ci-build ]; then - circleci-agent step halt - fi - -step-ts-compile: &step-ts-compile - run: - name: Run TS/JS compile on doc only change - command: | - cd src - ninja -C out/Default electron:default_app_js -j $NUMBER_OF_NINJA_PROCESSES - ninja -C out/Default electron:electron_js2c -j $NUMBER_OF_NINJA_PROCESSES - -# Lists of steps. steps-lint: &steps-lint steps: - *step-checkout-electron @@ -1078,551 +118,8 @@ steps-lint: &steps-lint cd src/electron node script/yarn tsc -p tsconfig.script.json -steps-electron-gn-check: &steps-electron-gn-check - steps: - - attach_workspace: - at: . - - *step-maybe-early-exit-doc-only-change - - *step-depot-tools-add-to-path - - *step-setup-env-for-build - - *step-setup-goma-for-build - - *step-gn-gen-default - - *step-gn-check - -steps-electron-ts-compile-for-doc-change: &steps-electron-ts-compile-for-doc-change - steps: - # Checkout - Copied ffrom steps-checkout - - *step-checkout-electron - - *step-check-for-doc-only-change - - *step-maybe-early-exit-no-doc-change - - *step-depot-tools-get - - *step-depot-tools-add-to-path - - *step-restore-brew-cache - - *step-install-gnutar-on-mac - - *step-get-more-space-on-mac - - *step-generate-deps-hash - - *step-touch-sync-done - - maybe-restore-portaled-src-cache - - *step-maybe-restore-git-cache - - *step-set-git-cache-path - # This sync call only runs if .circle-sync-done is an EMPTY file - - *step-gclient-sync - # These next few steps reset Electron to the correct commit regardless of which cache was restored - - run: - name: Wipe Electron - command: rm -rf src/electron - - *step-checkout-electron - - *step-run-electron-only-hooks - - *step-generate-deps-hash-cleanly - - *step-mark-sync-done - - *step-minimize-workspace-size-from-checkout - - - *step-depot-tools-add-to-path - - *step-setup-env-for-build - - *step-setup-goma-for-build - - *step-get-more-space-on-mac - - *step-install-npm-deps-on-mac - - *step-fix-sync - - *step-gn-gen-default - - #Compile ts/js to verify doc change didn't break anything - - *step-ts-compile - -steps-native-tests: &steps-native-tests - steps: - - attach_workspace: - at: . - - *step-depot-tools-add-to-path - - *step-setup-env-for-build - - *step-setup-goma-for-build - - *step-gn-gen-default - - - run: - name: Build tests - command: | - cd src - ninja -C out/Default $BUILD_TARGET - - *step-show-goma-stats - - - *step-setup-linux-for-headless-testing - - run: - name: Run tests - command: | - mkdir test_results - python src/electron/script/native-tests.py run \ - --config $TESTS_CONFIG \ - --tests-dir src/out/Default \ - --output-dir test_results \ - $TESTS_ARGS - - - store_artifacts: - path: test_results - destination: test_results # Put it in the root folder. - - store_test_results: - path: test_results - -steps-verify-ffmpeg: &steps-verify-ffmpeg - steps: - - attach_workspace: - at: . - - *step-depot-tools-add-to-path - - *step-electron-dist-unzip - - *step-ffmpeg-unzip - - *step-setup-linux-for-headless-testing - - - *step-verify-ffmpeg - - *step-maybe-notify-slack-failure - -steps-tests: &steps-tests - steps: - - attach_workspace: - at: . - - *step-maybe-early-exit-doc-only-change - - *step-depot-tools-add-to-path - - *step-electron-dist-unzip - - *step-mksnapshot-unzip - - *step-chromedriver-unzip - - *step-setup-linux-for-headless-testing - - *step-restore-brew-cache - - *step-fix-known-hosts-linux - - *step-install-signing-cert-on-mac - - - run: - name: Run Electron tests - environment: - MOCHA_REPORTER: mocha-multi-reporters - ELECTRON_TEST_RESULTS_DIR: junit - MOCHA_MULTI_REPORTERS: mocha-junit-reporter, tap - ELECTRON_DISABLE_SECURITY_WARNINGS: 1 - command: | - cd src - if [ "$IS_ASAN" == "1" ]; then - ASAN_SYMBOLIZE="$PWD/tools/valgrind/asan/asan_symbolize.py --executable-path=$PWD/out/Default/electron" - export ASAN_OPTIONS="symbolize=0 handle_abort=1" - export G_SLICE=always-malloc - export NSS_DISABLE_ARENA_FREE_LIST=1 - export NSS_DISABLE_UNLOAD=1 - export LLVM_SYMBOLIZER_PATH=$PWD/third_party/llvm-build/Release+Asserts/bin/llvm-symbolizer - export MOCHA_TIMEOUT=180000 - echo "Piping output to ASAN_SYMBOLIZE ($ASAN_SYMBOLIZE)" - (cd electron && node script/yarn test --runners=main --trace-uncaught --enable-logging --files $(circleci tests glob spec-main/*-spec.ts | circleci tests split --split-by=timings)) 2>&1 | $ASAN_SYMBOLIZE - (cd electron && node script/yarn test --runners=remote --trace-uncaught --enable-logging --files $(circleci tests glob spec/*-spec.js | circleci tests split --split-by=timings)) 2>&1 | $ASAN_SYMBOLIZE - else - if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then - export ELECTRON_SKIP_NATIVE_MODULE_TESTS=true - (cd electron && node script/yarn test --runners=main --trace-uncaught --enable-logging) - (cd electron && node script/yarn test --runners=remote --trace-uncaught --enable-logging) - else - if [ "$TARGET_ARCH" == "ia32" ]; then - npm_config_arch=x64 node electron/node_modules/dugite/script/download-git.js - fi - (cd electron && node script/yarn test --runners=main --trace-uncaught --enable-logging --files $(circleci tests glob spec-main/*-spec.ts | circleci tests split --split-by=timings)) - (cd electron && node script/yarn test --runners=remote --trace-uncaught --enable-logging --files $(circleci tests glob spec/*-spec.js | circleci tests split --split-by=timings)) - fi - fi - - run: - name: Check test results existence - command: | - cd src - - # Check if test results exist and are not empty. - if [ ! -s "junit/test-results-remote.xml" ]; then - exit 1 - fi - if [ ! -s "junit/test-results-main.xml" ]; then - exit 1 - fi - - store_test_results: - path: src/junit - - - *step-verify-mksnapshot - - *step-verify-chromedriver - - - *step-maybe-notify-slack-failure - - - *step-maybe-cleanup-arm64-mac - -steps-test-nan: &steps-test-nan - steps: - - attach_workspace: - at: . - - *step-maybe-early-exit-doc-only-change - - *step-depot-tools-add-to-path - - *step-electron-dist-unzip - - *step-setup-linux-for-headless-testing - - *step-fix-known-hosts-linux - - run: - name: Run Nan Tests - command: | - cd src - node electron/script/nan-spec-runner.js - -steps-test-node: &steps-test-node - steps: - - attach_workspace: - at: . - - *step-maybe-early-exit-doc-only-change - - *step-depot-tools-add-to-path - - *step-electron-dist-unzip - - *step-setup-linux-for-headless-testing - - *step-fix-known-hosts-linux - - run: - name: Run Node Tests - command: | - cd src - node electron/script/node-spec-runner.js --default --jUnitDir=junit - - store_test_results: - path: src/junit - -# Command Aliases -commands: - maybe-restore-portaled-src-cache: - parameters: - halt-if-successful: - type: boolean - default: false - steps: - - run: - name: Prepare for cross-OS sync restore - command: | - sudo mkdir -p /var/portal - sudo chown -R $(id -u):$(id -g) /var/portal - - when: - condition: << parameters.halt-if-successful >> - steps: - - *step-maybe-restore-src-cache-marker - - run: - name: Halt the job early if the src cache exists - command: | - if [ -f ".src-cache-marker" ]; then - circleci-agent step halt - fi - - *step-maybe-restore-src-cache - - run: - name: Fix the src cache restore point on macOS - command: | - if [ -d "/var/portal/src" ]; then - echo Relocating Cache - rm -rf src - mv /var/portal/src ./ - fi - - move_and_store_all_artifacts: - steps: - - run: - name: Move all generated artifacts to upload folder - command: | - rm -rf generated_artifacts - mkdir generated_artifacts - mv_if_exist() { - if [ -f "$1" ] || [ -d "$1" ]; then - echo Storing $1 - mv $1 generated_artifacts - else - echo Skipping $1 - It is not present on disk - fi - } - mv_if_exist src/out/Default/dist.zip - mv_if_exist src/out/Default/shell_browser_ui_unittests - mv_if_exist src/out/Default/gen/node_headers.tar.gz - mv_if_exist src/out/Default/symbols.zip - mv_if_exist src/out/Default/mksnapshot.zip - mv_if_exist src/out/Default/chromedriver.zip - mv_if_exist src/out/ffmpeg/ffmpeg.zip - mv_if_exist src/out/Default/hunspell_dictionaries.zip - mv_if_exist src/cross-arch-snapshots - mv_if_exist src/out/electron_ninja_log - when: always - - store_artifacts: - path: generated_artifacts - destination: ./ - - store_artifacts: - path: generated_artifacts/cross-arch-snapshots - destination: cross-arch-snapshots - - checkout-from-cache: - steps: - - *step-checkout-electron - - *step-maybe-early-exit-doc-only-change - - *step-depot-tools-get - - *step-depot-tools-add-to-path - - *step-generate-deps-hash - - maybe-restore-portaled-src-cache - - run: - name: Ensure src checkout worked - command: | - if [ ! -d "src/third_party/blink" ]; then - echo src cache was not restored for some reason, idk what happened here... - exit 1 - fi - - run: - name: Wipe Electron - command: rm -rf src/electron - - *step-checkout-electron - - *step-run-electron-only-hooks - - *step-generate-deps-hash-cleanly - electron-build: - parameters: - attach: - type: boolean - default: false - persist: - type: boolean - default: true - persist-checkout: - type: boolean - default: false - checkout: - type: boolean - default: true - checkout-and-assume-cache: - type: boolean - default: false - save-git-cache: - type: boolean - default: false - checkout-to-create-src-cache: - type: boolean - default: false - build: - type: boolean - default: true - use-out-cache: - type: boolean - default: true - restore-src-cache: - type: boolean - default: true - build-nonproprietary-ffmpeg: - type: boolean - default: true - steps: - - when: - condition: << parameters.attach >> - steps: - - attach_workspace: - at: . - - *step-maybe-early-exit-doc-only-change - - run: rm -rf src/electron - - *step-restore-brew-cache - - *step-install-gnutar-on-mac - - *step-save-brew-cache - - when: - condition: << parameters.checkout-and-assume-cache >> - steps: - - checkout-from-cache - - when: - condition: << parameters.checkout >> - steps: - # Checkout - Copied ffrom steps-checkout - - *step-checkout-electron - - *step-check-for-doc-only-change - - *step-persist-doc-only-change - - *step-maybe-early-exit-doc-only-change - - *step-depot-tools-get - - *step-depot-tools-add-to-path - - *step-get-more-space-on-mac - - *step-generate-deps-hash - - *step-touch-sync-done - - when: - condition: << parameters.restore-src-cache >> - steps: - - maybe-restore-portaled-src-cache: - halt-if-successful: << parameters.checkout-to-create-src-cache >> - - *step-maybe-restore-git-cache - - *step-set-git-cache-path - # This sync call only runs if .circle-sync-done is an EMPTY file - - *step-gclient-sync - - store_artifacts: - path: patches - - when: - condition: << parameters.save-git-cache >> - steps: - - *step-save-git-cache - # These next few steps reset Electron to the correct commit regardless of which cache was restored - - run: - name: Wipe Electron - command: rm -rf src/electron - - *step-checkout-electron - - *step-run-electron-only-hooks - - *step-generate-deps-hash-cleanly - - *step-mark-sync-done - - *step-minimize-workspace-size-from-checkout - - *step-delete-git-directories - - when: - condition: << parameters.persist-checkout >> - steps: - - persist_to_workspace: - root: . - paths: - - depot_tools - - src - - when: - condition: << parameters.checkout-to-create-src-cache >> - steps: - - run: - name: Move src folder to the cross-OS portal - command: | - sudo mkdir -p /var/portal - sudo chown -R $(id -u):$(id -g) /var/portal - mv ./src /var/portal - - *step-save-src-cache - - *step-make-src-cache-marker - - *step-save-src-cache-marker - - - when: - condition: << parameters.build >> - steps: - - *step-depot-tools-add-to-path - - *step-setup-env-for-build - - *step-setup-goma-for-build - - *step-get-more-space-on-mac - - *step-fix-sync - - *step-delete-git-directories - - # Electron app - - when: - condition: << parameters.use-out-cache >> - steps: - - *step-restore-out-cache - - *step-gn-gen-default - - *step-electron-build - - *step-maybe-electron-dist-strip - - *step-electron-dist-build - - # Native test targets - - *step-native-unittests-build - - # Node.js headers - - *step-nodejs-headers-build - - - *step-show-goma-stats - - # mksnapshot - - *step-mksnapshot-build - - *step-maybe-cross-arch-snapshot - - # chromedriver - - *step-electron-chromedriver-build - - - when: - condition: << parameters.build-nonproprietary-ffmpeg >> - steps: - # ffmpeg - - *step-ffmpeg-gn-gen - - *step-ffmpeg-build - - # hunspell - - *step-hunspell-build - - # Save all data needed for a further tests run. - - when: - condition: << parameters.persist >> - steps: - - *step-persist-data-for-tests - - - when: - condition: << parameters.build >> - steps: - - *step-maybe-generate-breakpad-symbols - - *step-maybe-zip-symbols - - - when: - condition: << parameters.build >> - steps: - - move_and_store_all_artifacts - - run: - name: Remove the big things on macOS, this seems to be better on average - command: | - if [ "`uname`" == "Darwin" ]; then - mkdir -p src/out/Default - cd src/out/Default - find . -type f -size +50M -delete - mkdir -p gen/electron - cd gen/electron - # These files do not seem to like being in a cache, let us remove them - find . -type f -name '*_pkg_info' -delete - fi - - when: - condition: << parameters.use-out-cache >> - steps: - - *step-save-out-cache - - - *step-maybe-notify-slack-failure - - electron-publish: - parameters: - attach: - type: boolean - default: false - checkout: - type: boolean - default: true - steps: - - when: - condition: << parameters.attach >> - steps: - - attach_workspace: - at: . - - when: - condition: << parameters.checkout >> - steps: - - *step-depot-tools-get - - *step-depot-tools-add-to-path - - *step-restore-brew-cache - - *step-get-more-space-on-mac - - when: - condition: << parameters.checkout >> - steps: - - *step-checkout-electron - - *step-touch-sync-done - - *step-maybe-restore-git-cache - - *step-set-git-cache-path - - *step-gclient-sync - - *step-delete-git-directories - - *step-minimize-workspace-size-from-checkout - - *step-fix-sync - - *step-setup-env-for-build - - *step-setup-goma-for-build - - *step-gn-gen-default - - # Electron app - - *step-electron-build - - *step-show-goma-stats - - *step-maybe-generate-breakpad-symbols - - *step-maybe-electron-dist-strip - - *step-electron-dist-build - - *step-maybe-zip-symbols - - # mksnapshot - - *step-mksnapshot-build - - # chromedriver - - *step-electron-chromedriver-build - - # Node.js headers - - *step-nodejs-headers-build - - # ffmpeg - - *step-ffmpeg-gn-gen - - *step-ffmpeg-build - - # hunspell - - *step-hunspell-build - - # libcxx - - *step-maybe-generate-libcxx - - # typescript defs - - *step-maybe-generate-typescript-defs - - # Publish - - *step-electron-publish - - move_and_store_all_artifacts - -# List of all jobs. +# List of always run jobs. jobs: - # Layer 0: Lint. Standalone. lint: executor: name: linux-docker @@ -1631,971 +128,18 @@ jobs: <<: *env-linux-medium <<: *steps-lint - ts-compile-doc-change: - executor: - name: linux-docker - size: medium - environment: - <<: *env-linux-medium - <<: *env-testing-build - <<: *steps-electron-ts-compile-for-doc-change - - # Layer 1: Checkout. - linux-checkout-for-workspace: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' - steps: - - electron-build: - persist: false - build: false - checkout: true - persist-checkout: true - - linux-make-src-cache: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' - steps: - - electron-build: - persist: false - build: false - checkout: true - save-git-cache: true - checkout-to-create-src-cache: true - - linux-checkout-for-native-tests: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_pyyaml=True' - steps: - - electron-build: - persist: false - build: false - checkout: true - persist-checkout: true - - linux-checkout-for-native-tests-with-no-patches: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - GCLIENT_EXTRA_ARGS: '--custom-var=apply_patches=False --custom-var=checkout_pyyaml=True' - steps: - - electron-build: - persist: false - build: false - checkout: true - persist-checkout: true - - mac-checkout: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - <<: *env-testing-build - <<: *env-macos-build - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' - steps: - - electron-build: - persist: false - build: false - checkout: true - persist-checkout: true - restore-src-cache: false - - mac-checkout-for-workspace: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - <<: *env-testing-build - <<: *env-macos-build - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' - steps: - - electron-build: - persist: false - build: false - checkout: true - persist-checkout: true - - mac-make-src-cache: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - <<: *env-testing-build - <<: *env-macos-build - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' - steps: - - electron-build: - persist: false - build: false - checkout: true - save-git-cache: true - checkout-to-create-src-cache: true - - # Layer 2: Builds. - linux-x64-testing: - executor: linux-docker - environment: - <<: *env-global - <<: *env-testing-build - <<: *env-ninja-status - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' - steps: - - electron-build: - persist: true - checkout: true - use-out-cache: false - - linux-x64-testing-asan: - executor: linux-docker - environment: - <<: *env-global - <<: *env-testing-build - <<: *env-ninja-status - CHECK_DIST_MANIFEST: '0' - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' - GN_EXTRA_ARGS: 'is_asan = true' - steps: - - electron-build: - persist: true - checkout: true - use-out-cache: false - build-nonproprietary-ffmpeg: false - - linux-x64-testing-no-run-as-node: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - <<: *env-testing-build - <<: *env-ninja-status - <<: *env-disable-run-as-node - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' - steps: - - electron-build: - persist: false - checkout: true - use-out-cache: false - - linux-x64-testing-gn-check: - executor: - name: linux-docker - size: medium - environment: - <<: *env-linux-medium - <<: *env-testing-build - <<: *steps-electron-gn-check - - linux-x64-release: - executor: linux-docker - environment: - <<: *env-linux-2xlarge-release - <<: *env-release-build - <<: *env-send-slack-notifications - <<: *env-ninja-status - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' - steps: - - electron-build: - persist: true - checkout: true - - linux-x64-publish: - executor: linux-docker - environment: - <<: *env-linux-2xlarge-release - <<: *env-release-build - UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> - <<: *env-ninja-status - steps: - - run: echo running - - when: - condition: - or: - - equal: ["all", << pipeline.parameters.linux-publish-arch-limit >>] - - equal: ["x64", << pipeline.parameters.linux-publish-arch-limit >>] - steps: - - electron-publish: - attach: false - checkout: true - - linux-ia32-testing: - executor: linux-docker - environment: - <<: *env-global - <<: *env-ia32 - <<: *env-testing-build - <<: *env-ninja-status - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' - steps: - - electron-build: - persist: true - checkout: true - use-out-cache: false - - linux-ia32-release: - executor: linux-docker - environment: - <<: *env-linux-2xlarge-release - <<: *env-ia32 - <<: *env-release-build - <<: *env-send-slack-notifications - <<: *env-ninja-status - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' - steps: - - electron-build: - persist: true - checkout: true - - linux-ia32-publish: - executor: linux-docker - environment: - <<: *env-linux-2xlarge-release - <<: *env-ia32 - <<: *env-release-build - <<: *env-32bit-release - UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> - <<: *env-ninja-status - steps: - - run: echo running - - when: - condition: - or: - - equal: ["all", << pipeline.parameters.linux-publish-arch-limit >>] - - equal: ["ia32", << pipeline.parameters.linux-publish-arch-limit >>] - steps: - - electron-publish: - attach: false - checkout: true - - linux-arm-testing: - executor: linux-docker - environment: - <<: *env-global - <<: *env-arm - <<: *env-testing-build - <<: *env-ninja-status - TRIGGER_ARM_TEST: true - GENERATE_CROSS_ARCH_SNAPSHOT: true - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' - steps: - - electron-build: - persist: true - checkout: true - use-out-cache: false - - linux-arm-release: - executor: linux-docker - environment: - <<: *env-linux-2xlarge-release - <<: *env-arm - <<: *env-release-build - <<: *env-send-slack-notifications - <<: *env-ninja-status - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' - steps: - - electron-build: - persist: false - checkout: true - - linux-arm-publish: - executor: linux-docker - environment: - <<: *env-linux-2xlarge-release - <<: *env-arm - <<: *env-release-build - <<: *env-32bit-release - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True' - UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> - <<: *env-ninja-status - steps: - - run: echo running - - when: - condition: - or: - - equal: ["all", << pipeline.parameters.linux-publish-arch-limit >>] - - equal: ["arm", << pipeline.parameters.linux-publish-arch-limit >>] - steps: - - electron-publish: - attach: false - checkout: true - - linux-arm64-testing: - executor: linux-docker - environment: - <<: *env-global - <<: *env-arm64 - <<: *env-testing-build - <<: *env-ninja-status - TRIGGER_ARM_TEST: true - GENERATE_CROSS_ARCH_SNAPSHOT: true - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' - steps: - - electron-build: - persist: true - checkout: true - use-out-cache: false - - linux-arm64-testing-gn-check: - executor: - name: linux-docker - size: medium - environment: - <<: *env-linux-medium - <<: *env-arm64 - <<: *env-testing-build - <<: *steps-electron-gn-check - - linux-arm64-release: - executor: linux-docker - environment: - <<: *env-linux-2xlarge-release - <<: *env-arm64 - <<: *env-release-build - <<: *env-send-slack-notifications - <<: *env-ninja-status - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' - steps: - - electron-build: - persist: false - checkout: true - - linux-arm64-publish: - executor: linux-docker - environment: - <<: *env-linux-2xlarge-release - <<: *env-arm64 - <<: *env-release-build - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True' - UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> - <<: *env-ninja-status - steps: - - run: echo running - - when: - condition: - or: - - equal: ["all", << pipeline.parameters.linux-publish-arch-limit >>] - - equal: ["arm64", << pipeline.parameters.linux-publish-arch-limit >>] - steps: - - electron-publish: - attach: false - checkout: true - - osx-testing-x64: - executor: macos - environment: - <<: *env-mac-large - <<: *env-testing-build - <<: *env-ninja-status - <<: *env-macos-build - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' - steps: - - electron-build: - persist: true - checkout: false - checkout-and-assume-cache: true - attach: true - - osx-testing-x64-gn-check: - executor: - name: macos - size: medium - environment: - <<: *env-machine-mac - <<: *env-testing-build - <<: *steps-electron-gn-check - - osx-publish-x64-skip-checkout: - executor: macos - environment: - <<: *env-mac-large-release - <<: *env-release-build - UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> - <<: *env-ninja-status - steps: - - run: echo running - - when: - condition: - or: - - equal: ["all", << pipeline.parameters.macos-publish-arch-limit >>] - - equal: ["osx-x64", << pipeline.parameters.macos-publish-arch-limit >>] - steps: - - electron-publish: - attach: true - checkout: false - - osx-publish-arm64-skip-checkout: - executor: macos - environment: - <<: *env-mac-large-release - <<: *env-release-build - <<: *env-apple-silicon - UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> - <<: *env-ninja-status - steps: - - run: echo running - - when: - condition: - or: - - equal: ["all", << pipeline.parameters.macos-publish-arch-limit >>] - - equal: ["osx-arm64", << pipeline.parameters.macos-publish-arch-limit >>] - steps: - - electron-publish: - attach: true - checkout: false - - osx-testing-arm64: - executor: macos - environment: - <<: *env-mac-large - <<: *env-testing-build - <<: *env-ninja-status - <<: *env-macos-build - <<: *env-apple-silicon - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' - GENERATE_CROSS_ARCH_SNAPSHOT: true - steps: - - electron-build: - persist: true - checkout: false - checkout-and-assume-cache: true - attach: true - - mas-testing-x64: - executor: macos - environment: - <<: *env-mac-large - <<: *env-mas - <<: *env-testing-build - <<: *env-ninja-status - <<: *env-macos-build - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' - steps: - - electron-build: - persist: true - checkout: false - checkout-and-assume-cache: true - attach: true - - mas-testing-x64-gn-check: - executor: - name: macos - size: medium - environment: - <<: *env-machine-mac - <<: *env-mas - <<: *env-testing-build - <<: *steps-electron-gn-check - - mas-publish-x64-skip-checkout: - executor: macos - environment: - <<: *env-mac-large-release - <<: *env-mas - <<: *env-release-build - UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> - steps: - - run: echo running - - when: - condition: - or: - - equal: ["all", << pipeline.parameters.macos-publish-arch-limit >>] - - equal: ["mas-x64", << pipeline.parameters.macos-publish-arch-limit >>] - steps: - - electron-publish: - attach: true - checkout: false - - mas-publish-arm64-skip-checkout: - executor: macos - environment: - <<: *env-mac-large-release - <<: *env-mas-apple-silicon - <<: *env-release-build - UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> - <<: *env-ninja-status - steps: - - run: echo running - - when: - condition: - or: - - equal: ["all", << pipeline.parameters.macos-publish-arch-limit >>] - - equal: ["mas-arm64", << pipeline.parameters.macos-publish-arch-limit >>] - steps: - - electron-publish: - attach: true - checkout: false - - mas-testing-arm64: - executor: macos - environment: - <<: *env-mac-large - <<: *env-testing-build - <<: *env-ninja-status - <<: *env-macos-build - <<: *env-mas-apple-silicon - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' - GENERATE_CROSS_ARCH_SNAPSHOT: true - steps: - - electron-build: - persist: true - checkout: false - checkout-and-assume-cache: true - attach: true - - # Layer 3: Tests. - linux-x64-unittests: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - <<: *env-unittests - <<: *env-headless-testing - <<: *steps-native-tests - - linux-x64-disabled-unittests: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - <<: *env-unittests - <<: *env-headless-testing - TESTS_ARGS: '--only-disabled-tests' - <<: *steps-native-tests - - linux-x64-chromium-unittests: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - <<: *env-unittests - <<: *env-headless-testing - TESTS_ARGS: '--include-disabled-tests' - <<: *steps-native-tests - - linux-x64-browsertests: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - <<: *env-browsertests - <<: *env-testing-build - <<: *env-headless-testing - <<: *steps-native-tests - - linux-x64-testing-tests: - executor: - name: linux-docker - size: medium - environment: - <<: *env-linux-medium - <<: *env-headless-testing - <<: *env-stack-dumping - parallelism: 3 - <<: *steps-tests - - linux-x64-testing-asan-tests: - executor: - name: linux-docker - size: xlarge - environment: - <<: *env-linux-medium - <<: *env-headless-testing - <<: *env-stack-dumping - IS_ASAN: '1' - DISABLE_CRASH_REPORTER_TESTS: '1' - parallelism: 3 - <<: *steps-tests - - linux-x64-testing-nan: - executor: - name: linux-docker - size: medium - environment: - <<: *env-linux-medium - <<: *env-headless-testing - <<: *env-stack-dumping - <<: *steps-test-nan - - linux-x64-testing-node: - executor: linux-docker - environment: - <<: *env-linux-medium - <<: *env-headless-testing - <<: *env-stack-dumping - <<: *steps-test-node - - linux-x64-release-tests: - executor: - name: linux-docker - size: medium - environment: - <<: *env-linux-medium - <<: *env-headless-testing - <<: *env-send-slack-notifications - <<: *steps-tests - - linux-x64-verify-ffmpeg: - executor: - name: linux-docker - size: medium - environment: - <<: *env-linux-medium - <<: *env-headless-testing - <<: *env-send-slack-notifications - <<: *steps-verify-ffmpeg - - linux-ia32-testing-tests: - executor: - name: linux-docker - size: medium - environment: - <<: *env-linux-medium - <<: *env-ia32 - <<: *env-headless-testing - <<: *env-stack-dumping - parallelism: 3 - <<: *steps-tests - - linux-ia32-testing-nan: - executor: - name: linux-docker - size: medium - environment: - <<: *env-linux-medium - <<: *env-ia32 - <<: *env-headless-testing - <<: *env-stack-dumping - <<: *steps-test-nan - - linux-ia32-testing-node: - executor: linux-docker - environment: - <<: *env-linux-medium - <<: *env-ia32 - <<: *env-headless-testing - <<: *env-stack-dumping - <<: *steps-test-node - - linux-ia32-release-tests: - executor: - name: linux-docker - size: medium - environment: - <<: *env-linux-medium - <<: *env-ia32 - <<: *env-headless-testing - <<: *env-send-slack-notifications - <<: *steps-tests - - linux-ia32-verify-ffmpeg: - executor: - name: linux-docker - size: medium - environment: - <<: *env-linux-medium - <<: *env-ia32 - <<: *env-headless-testing - <<: *env-send-slack-notifications - <<: *steps-verify-ffmpeg - - linux-arm-testing-tests: - executor: linux-arm - environment: - <<: *env-arm - <<: *env-global - <<: *env-headless-testing - <<: *env-stack-dumping - <<: *steps-tests - - linux-arm64-testing-tests: - executor: linux-arm64 - environment: - <<: *env-arm64 - <<: *env-global - <<: *env-headless-testing - <<: *env-stack-dumping - <<: *steps-tests - - osx-testing-x64-tests: - executor: - name: macos - size: medium - environment: - <<: *env-mac-large - <<: *env-stack-dumping - parallelism: 2 - <<: *steps-tests - - osx-testing-arm64-tests: - executor: apple-silicon - environment: - <<: *env-mac-large - <<: *env-stack-dumping - <<: *env-apple-silicon - <<: *steps-tests - - mas-testing-x64-tests: - executor: - name: macos - size: medium - environment: - <<: *env-mac-large - <<: *env-stack-dumping - parallelism: 2 - <<: *steps-tests - - mas-testing-arm64-tests: - executor: apple-silicon - environment: - <<: *env-mac-large - <<: *env-stack-dumping - <<: *env-apple-silicon - <<: *steps-tests - - # Layer 4: Summary. - linux-release-summary: - executor: - name: linux-docker - size: medium - environment: - <<: *env-linux-medium - <<: *env-send-slack-notifications - steps: - - *step-maybe-notify-slack-success - +# Initial setup workflow workflows: - version: 2.1 - - publish-linux: - when: << pipeline.parameters.run-linux-publish >> - jobs: - - linux-x64-publish: - context: release-env - - linux-ia32-publish: - context: release-env - - linux-arm-publish: - context: release-env - - linux-arm64-publish: - context: release-env - - publish-macos: - when: << pipeline.parameters.run-macos-publish >> - jobs: - - mac-checkout - - osx-publish-x64-skip-checkout: - requires: - - mac-checkout - context: release-env - - mas-publish-x64-skip-checkout: - requires: - - mac-checkout - context: release-env - - osx-publish-arm64-skip-checkout: - requires: - - mac-checkout - context: release-env - - mas-publish-arm64-skip-checkout: - requires: - - mac-checkout - context: release-env - lint: - when: << pipeline.parameters.run-lint >> jobs: + # Job inherited from path-filtering orb + - path-filtering/filter: + base-revision: main + # Params for mapping; `path-to-test parameter-to-set value-for-parameter` for each row + mapping: | + ^((?!docs/).)*$ run-build-mac true + ^((?!docs/).)*$ run-build-linux true + docs/.* run-docs-only true + ^((?!docs/).)*$ run-docs-only false + config-path: .circleci/build_config.yml - lint - - build-linux: - when: << pipeline.parameters.run-build-linux >> - jobs: - - linux-checkout-for-workspace - - linux-make-src-cache - - - linux-x64-testing - - linux-x64-testing-asan - - linux-x64-testing-no-run-as-node - - linux-x64-testing-gn-check: - requires: - - linux-checkout-for-workspace - - linux-x64-testing-tests: - requires: - - linux-x64-testing - - linux-x64-testing-asan-tests: - requires: - - linux-x64-testing-asan - - linux-x64-testing-nan: - requires: - - linux-x64-testing - - linux-x64-testing-node: - requires: - - linux-x64-testing - - - linux-ia32-testing - - linux-ia32-testing-tests: - requires: - - linux-ia32-testing - - linux-ia32-testing-nan: - requires: - - linux-ia32-testing - - linux-ia32-testing-node: - requires: - - linux-ia32-testing - - - linux-arm-testing - - linux-arm-testing-tests: - filters: - branches: - # Do not run this on forked pull requests - ignore: /pull\/[0-9]+/ - requires: - - linux-arm-testing - - - linux-arm64-testing - - linux-arm64-testing-tests: - filters: - branches: - # Do not run this on forked pull requests - ignore: /pull\/[0-9]+/ - requires: - - linux-arm64-testing - - - linux-arm64-testing-gn-check: - requires: - - linux-checkout-for-workspace - - ts-compile-doc-change - - build-mac: - when: << pipeline.parameters.run-build-mac >> - jobs: - - mac-checkout-for-workspace - - mac-make-src-cache - - - osx-testing-x64: - requires: - - mac-make-src-cache - - - osx-testing-x64-gn-check: - requires: - - mac-checkout-for-workspace - - - osx-testing-x64-tests: - requires: - - osx-testing-x64 - - - osx-testing-arm64: - requires: - - mac-make-src-cache - - - osx-testing-arm64-tests: - filters: - branches: - # Do not run this on forked pull requests - ignore: /pull\/[0-9]+/ - requires: - - osx-testing-arm64 - - - mas-testing-x64: - requires: - - mac-make-src-cache - - - mas-testing-x64-gn-check: - requires: - - mac-checkout-for-workspace - - - mas-testing-x64-tests: - requires: - - mas-testing-x64 - - - mas-testing-arm64: - requires: - - mac-make-src-cache - - - mas-testing-arm64-tests: - filters: - branches: - # Do not run this on forked pull requests - ignore: /pull\/[0-9]+/ - requires: - - mas-testing-arm64 - - nightly-linux-release-test: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - master - jobs: - - linux-checkout-for-workspace - - linux-make-src-cache - - - linux-x64-release - - linux-x64-release-tests: - requires: - - linux-x64-release - - linux-x64-verify-ffmpeg: - requires: - - linux-x64-release - - linux-release-summary: - name: linux-x64-release-summary - requires: - - linux-x64-release - - linux-x64-release-tests - - linux-x64-verify-ffmpeg - - - linux-ia32-release - - linux-ia32-release-tests: - requires: - - linux-ia32-release - - linux-ia32-verify-ffmpeg: - requires: - - linux-ia32-release - - linux-release-summary: - name: linux-ia32-release-summary - requires: - - linux-ia32-release - - linux-ia32-release-tests - - linux-ia32-verify-ffmpeg - - - linux-arm-release - - linux-release-summary: - name: linux-arm-release-summary - requires: - - linux-arm-release - - - linux-arm64-release - - linux-release-summary: - name: linux-arm64-release-summary - requires: - - linux-arm64-release - - # Various slow and non-essential checks we run only nightly. - # Sanitizer jobs should be added here. - linux-checks-nightly: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - master - jobs: - - linux-checkout-for-native-tests - - # TODO(alexeykuzmin): Enable it back. - # Tons of crashes right now, see - # https://circleci.com/gh/electron/electron/67463 -# - linux-x64-browsertests: -# requires: -# - linux-checkout-for-native-tests - - - linux-x64-unittests: - requires: - - linux-checkout-for-native-tests - - - linux-x64-disabled-unittests: - requires: - - linux-checkout-for-native-tests - - - linux-checkout-for-native-tests-with-no-patches - - - linux-x64-chromium-unittests: - requires: - - linux-checkout-for-native-tests-with-no-patches diff --git a/script/release/ci-release-build.js b/script/release/ci-release-build.js index 278f2e5c70..063f09594b 100644 --- a/script/release/ci-release-build.js +++ b/script/release/ci-release-build.js @@ -59,11 +59,7 @@ async function circleCIcall (targetBranch, workflowName, options) { console.log(`Triggering CircleCI to run build job: ${workflowName} on branch: ${targetBranch} with release flag.`); const buildRequest = { branch: targetBranch, - parameters: { - 'run-lint': false, - 'run-build-linux': false, - 'run-build-mac': false - } + parameters: {} }; if (options.ghRelease) { buildRequest.parameters['upload-to-s3'] = '0';