ci: Speed up release (#22126)
* build: speed up release builds by using workflows * Don't get depot-tools twice * Do not use source from cache * Checkout boto and requests for release builds * Make sure vendor dirs are persisted. * Use git cache for releases. * show sccache stats right after build * Make messages easier for sudowoodo to consume.
This commit is contained in:
John Kleinschmidt
GitHub
Родитель
8b89c3cade
Коммит
023a3ca722
|
|
@ -41,6 +41,14 @@ parameters:
|
|||
type: boolean
|
||||
default: false
|
||||
|
||||
run-linux-publish:
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
run-macos-publish:
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
# The config expects the following environment variables to be set:
|
||||
# - "SLACK_WEBHOOK" Slack hook URL to send notifications.
|
||||
#
|
||||
|
|
@ -1030,56 +1038,6 @@ steps-electron-ts-compile-for-doc-change: &steps-electron-ts-compile-for-doc-cha
|
|||
#Compile ts/js to verify doc change didn't break anything
|
||||
- *step-ts-compile
|
||||
|
||||
steps-electron-build-for-publish: &steps-electron-build-for-publish
|
||||
steps:
|
||||
- *step-checkout-electron
|
||||
- *step-depot-tools-get
|
||||
- *step-depot-tools-add-to-path
|
||||
- *step-restore-brew-cache
|
||||
- *step-get-more-space-on-mac
|
||||
- *step-gclient-sync
|
||||
- *step-setup-env-for-build
|
||||
- *step-delete-git-directories
|
||||
- *step-minimize-workspace-size-from-checkout
|
||||
- *step-gn-gen-default
|
||||
|
||||
# Electron app
|
||||
- *step-electron-build
|
||||
- *step-maybe-generate-breakpad-symbols
|
||||
- *step-maybe-electron-dist-strip
|
||||
- *step-electron-dist-build
|
||||
- *step-electron-dist-store
|
||||
- *step-maybe-zip-symbols
|
||||
- *step-symbols-store
|
||||
|
||||
# mksnapshot
|
||||
- *step-mksnapshot-build
|
||||
- *step-mksnapshot-store
|
||||
|
||||
# chromedriver
|
||||
- *step-electron-maybe-chromedriver-gn-gen
|
||||
- *step-electron-chromedriver-build
|
||||
- *step-electron-chromedriver-store
|
||||
|
||||
# Node.js headers
|
||||
- *step-nodejs-headers-build
|
||||
- *step-nodejs-headers-store
|
||||
|
||||
# ffmpeg
|
||||
- *step-ffmpeg-gn-gen
|
||||
- *step-ffmpeg-build
|
||||
- *step-ffmpeg-store
|
||||
|
||||
# hunspell
|
||||
- *step-hunspell-build
|
||||
- *step-hunspell-store
|
||||
|
||||
# typescript defs
|
||||
- *step-maybe-generate-typescript-defs
|
||||
|
||||
# Publish
|
||||
- *step-electron-publish
|
||||
|
||||
steps-chromedriver-build: &steps-chromedriver-build
|
||||
steps:
|
||||
- attach_workspace:
|
||||
|
|
@ -1304,6 +1262,12 @@ commands:
|
|||
use-out-cache:
|
||||
type: boolean
|
||||
default: true
|
||||
restore-src-cache:
|
||||
type: boolean
|
||||
default: true
|
||||
preserve-vendor-dirs:
|
||||
type: boolean
|
||||
default: false
|
||||
steps:
|
||||
- when:
|
||||
condition: << parameters.attach >>
|
||||
|
|
@ -1330,17 +1294,38 @@ commands:
|
|||
- *step-get-more-space-on-mac
|
||||
- *step-generate-deps-hash
|
||||
- *step-touch-sync-done
|
||||
- maybe-restore-portaled-src-cache
|
||||
- when:
|
||||
condition: << parameters.restore-src-cache >>
|
||||
steps:
|
||||
- maybe-restore-portaled-src-cache
|
||||
- *step-maybe-restore-git-cache
|
||||
- *step-set-git-cache-path
|
||||
# This sync call only runs if .circle-sync-done is an EMPTY file
|
||||
- *step-gclient-sync
|
||||
# These next few steps reset Electron to the correct commit regardless of which cache was restored
|
||||
- when:
|
||||
condition: << parameters.preserve-vendor-dirs >>
|
||||
steps:
|
||||
- run:
|
||||
name: Preserve vendor dirs for release
|
||||
command: |
|
||||
mv src/electron/vendor/boto .
|
||||
mv src/electron/vendor/requests .
|
||||
- run:
|
||||
name: Wipe Electron
|
||||
command: rm -rf src/electron
|
||||
- *step-checkout-electron
|
||||
- *step-run-electron-only-hooks
|
||||
- when:
|
||||
condition: << parameters.preserve-vendor-dirs >>
|
||||
steps:
|
||||
- run:
|
||||
name: Preserve vendor dirs for release
|
||||
command: |
|
||||
rm -rf src/electron/vendor/boto
|
||||
rm -rf src/electron/vendor/requests
|
||||
mv boto src/electron/vendor
|
||||
mv requests src/electron/vendor/requests
|
||||
- *step-generate-deps-hash-cleanly
|
||||
- *step-mark-sync-done
|
||||
- *step-minimize-workspace-size-from-checkout
|
||||
|
|
@ -1439,10 +1424,80 @@ commands:
|
|||
steps:
|
||||
- *step-save-out-cache
|
||||
|
||||
# Trigger tests on arm hardware if needed
|
||||
- *step-maybe-trigger-arm-test
|
||||
# Trigger tests on arm hardware if needed
|
||||
- *step-maybe-trigger-arm-test
|
||||
|
||||
- *step-maybe-notify-slack-failure
|
||||
- *step-maybe-notify-slack-failure
|
||||
|
||||
electron-publish:
|
||||
parameters:
|
||||
attach:
|
||||
type: boolean
|
||||
default: false
|
||||
checkout:
|
||||
type: boolean
|
||||
default: true
|
||||
steps:
|
||||
- when:
|
||||
condition: << parameters.attach >>
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- when:
|
||||
condition: << parameters.checkout >>
|
||||
steps:
|
||||
- *step-depot-tools-get
|
||||
- *step-depot-tools-add-to-path
|
||||
- *step-restore-brew-cache
|
||||
- *step-get-more-space-on-mac
|
||||
- when:
|
||||
condition: << parameters.checkout >>
|
||||
steps:
|
||||
- *step-checkout-electron
|
||||
- *step-gclient-sync
|
||||
- *step-delete-git-directories
|
||||
- *step-minimize-workspace-size-from-checkout
|
||||
- *step-fix-sync-on-mac
|
||||
- *step-setup-env-for-build
|
||||
- *step-gn-gen-default
|
||||
|
||||
# Electron app
|
||||
- *step-electron-build
|
||||
- *step-show-sccache-stats
|
||||
- *step-maybe-generate-breakpad-symbols
|
||||
- *step-maybe-electron-dist-strip
|
||||
- *step-electron-dist-build
|
||||
- *step-electron-dist-store
|
||||
- *step-maybe-zip-symbols
|
||||
- *step-symbols-store
|
||||
|
||||
# mksnapshot
|
||||
- *step-mksnapshot-build
|
||||
- *step-mksnapshot-store
|
||||
|
||||
# chromedriver
|
||||
- *step-electron-maybe-chromedriver-gn-gen
|
||||
- *step-electron-chromedriver-build
|
||||
- *step-electron-chromedriver-store
|
||||
|
||||
# Node.js headers
|
||||
- *step-nodejs-headers-build
|
||||
- *step-nodejs-headers-store
|
||||
|
||||
# ffmpeg
|
||||
- *step-ffmpeg-gn-gen
|
||||
- *step-ffmpeg-build
|
||||
- *step-ffmpeg-store
|
||||
|
||||
# hunspell
|
||||
- *step-hunspell-build
|
||||
- *step-hunspell-store
|
||||
|
||||
# typescript defs
|
||||
- *step-maybe-generate-typescript-defs
|
||||
|
||||
# Publish
|
||||
- *step-electron-publish
|
||||
|
||||
# List of all jobs.
|
||||
jobs:
|
||||
|
|
@ -1461,6 +1516,20 @@ jobs:
|
|||
<<: *steps-electron-ts-compile-for-doc-change
|
||||
|
||||
# Layer 1: Checkout.
|
||||
linux-checkout:
|
||||
<<: *machine-linux-2xlarge
|
||||
environment:
|
||||
<<: *env-linux-2xlarge
|
||||
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True --custom-var=checkout_boto=True --custom-var=checkout_requests=True'
|
||||
steps:
|
||||
- electron-build:
|
||||
persist: false
|
||||
build: false
|
||||
checkout: true
|
||||
persist-checkout: true
|
||||
restore-src-cache: false
|
||||
preserve-vendor-dirs: true
|
||||
|
||||
linux-checkout-fast:
|
||||
<<: *machine-linux-2xlarge
|
||||
environment:
|
||||
|
|
@ -1504,6 +1573,22 @@ jobs:
|
|||
checkout: true
|
||||
persist-checkout: true
|
||||
|
||||
mac-checkout:
|
||||
<<: *machine-linux-2xlarge
|
||||
environment:
|
||||
<<: *env-linux-2xlarge
|
||||
<<: *env-testing-build
|
||||
<<: *env-macos-build
|
||||
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac --custom-var=checkout_boto=True --custom-var=checkout_requests=True'
|
||||
steps:
|
||||
- electron-build:
|
||||
persist: false
|
||||
build: false
|
||||
checkout: true
|
||||
persist-checkout: true
|
||||
restore-src-cache: false
|
||||
preserve-vendor-dirs: true
|
||||
|
||||
mac-checkout-fast:
|
||||
<<: *machine-linux-2xlarge
|
||||
environment:
|
||||
|
|
@ -1591,7 +1676,22 @@ jobs:
|
|||
<<: *env-release-build
|
||||
<<: *env-enable-sccache
|
||||
UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
|
||||
<<: *steps-electron-build-for-publish
|
||||
steps:
|
||||
- electron-publish:
|
||||
attach: false
|
||||
checkout: true
|
||||
|
||||
linux-x64-publish-skip-checkout:
|
||||
<<: *machine-linux-2xlarge
|
||||
environment:
|
||||
<<: *env-linux-2xlarge-release
|
||||
<<: *env-release-build
|
||||
<<: *env-enable-sccache
|
||||
UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
|
||||
steps:
|
||||
- electron-publish:
|
||||
attach: true
|
||||
checkout: false
|
||||
|
||||
linux-ia32-testing:
|
||||
<<: *machine-linux-2xlarge
|
||||
|
|
@ -1640,7 +1740,24 @@ jobs:
|
|||
<<: *env-enable-sccache
|
||||
<<: *env-32bit-release
|
||||
UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
|
||||
<<: *steps-electron-build-for-publish
|
||||
steps:
|
||||
- electron-publish:
|
||||
attach: false
|
||||
checkout: true
|
||||
|
||||
linux-ia32-publish-skip-checkout:
|
||||
<<: *machine-linux-2xlarge
|
||||
environment:
|
||||
<<: *env-linux-2xlarge-release
|
||||
<<: *env-ia32
|
||||
<<: *env-release-build
|
||||
<<: *env-enable-sccache
|
||||
<<: *env-32bit-release
|
||||
UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
|
||||
steps:
|
||||
- electron-publish:
|
||||
attach: true
|
||||
checkout: false
|
||||
|
||||
linux-arm-testing:
|
||||
<<: *machine-linux-2xlarge
|
||||
|
|
@ -1690,7 +1807,24 @@ jobs:
|
|||
<<: *env-32bit-release
|
||||
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_boto=True --custom-var=checkout_requests=True'
|
||||
UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
|
||||
<<: *steps-electron-build-for-publish
|
||||
steps:
|
||||
- electron-publish:
|
||||
attach: false
|
||||
checkout: true
|
||||
|
||||
linux-arm-publish-skip-checkout:
|
||||
<<: *machine-linux-2xlarge
|
||||
environment:
|
||||
<<: *env-linux-2xlarge-release
|
||||
<<: *env-arm
|
||||
<<: *env-release-build
|
||||
<<: *env-enable-sccache
|
||||
<<: *env-32bit-release
|
||||
UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
|
||||
steps:
|
||||
- electron-publish:
|
||||
attach: true
|
||||
checkout: false
|
||||
|
||||
linux-arm64-testing:
|
||||
<<: *machine-linux-2xlarge
|
||||
|
|
@ -1747,7 +1881,23 @@ jobs:
|
|||
<<: *env-enable-sccache
|
||||
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True --custom-var=checkout_boto=True --custom-var=checkout_requests=True'
|
||||
UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
|
||||
<<: *steps-electron-build-for-publish
|
||||
steps:
|
||||
- electron-publish:
|
||||
attach: false
|
||||
checkout: true
|
||||
|
||||
linux-arm64-publish-skip-checkout:
|
||||
<<: *machine-linux-2xlarge
|
||||
environment:
|
||||
<<: *env-linux-2xlarge-release
|
||||
<<: *env-arm64
|
||||
<<: *env-release-build
|
||||
<<: *env-enable-sccache
|
||||
UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
|
||||
steps:
|
||||
- electron-publish:
|
||||
attach: true
|
||||
checkout: false
|
||||
|
||||
osx-testing:
|
||||
<<: *machine-mac-large
|
||||
|
|
@ -1801,7 +1951,22 @@ jobs:
|
|||
<<: *env-enable-sccache
|
||||
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True'
|
||||
UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
|
||||
<<: *steps-electron-build-for-publish
|
||||
steps:
|
||||
- electron-publish:
|
||||
attach: false
|
||||
checkout: true
|
||||
|
||||
osx-publish-skip-checkout:
|
||||
<<: *machine-mac-large
|
||||
environment:
|
||||
<<: *env-mac-large-release
|
||||
<<: *env-release-build
|
||||
<<: *env-enable-sccache
|
||||
UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
|
||||
steps:
|
||||
- electron-publish:
|
||||
attach: true
|
||||
checkout: false
|
||||
|
||||
mas-testing:
|
||||
<<: *machine-mac-large
|
||||
|
|
@ -1859,7 +2024,23 @@ jobs:
|
|||
<<: *env-enable-sccache
|
||||
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True'
|
||||
UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
|
||||
<<: *steps-electron-build-for-publish
|
||||
steps:
|
||||
- electron-publish:
|
||||
attach: false
|
||||
checkout: true
|
||||
|
||||
mas-publish-skip-checkout:
|
||||
<<: *machine-mac-large
|
||||
environment:
|
||||
<<: *env-mac-large-release
|
||||
<<: *env-mas
|
||||
<<: *env-release-build
|
||||
<<: *env-enable-sccache
|
||||
UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
|
||||
steps:
|
||||
- electron-publish:
|
||||
attach: true
|
||||
checkout: false
|
||||
|
||||
# Layer 3: Tests.
|
||||
linux-x64-unittests:
|
||||
|
|
@ -2149,6 +2330,27 @@ workflows:
|
|||
# will need to be updated and there will most likely need to be changes to
|
||||
# sudowoodo
|
||||
|
||||
publish-linux:
|
||||
when: << pipeline.parameters.run-linux-publish >>
|
||||
jobs:
|
||||
- linux-checkout
|
||||
- linux-x64-publish-skip-checkout:
|
||||
requires:
|
||||
- linux-checkout
|
||||
context: release-env
|
||||
- linux-ia32-publish-skip-checkout:
|
||||
requires:
|
||||
- linux-checkout
|
||||
context: release-env
|
||||
- linux-arm-publish-skip-checkout:
|
||||
requires:
|
||||
- linux-checkout
|
||||
context: release-env
|
||||
- linux-arm64-publish-skip-checkout:
|
||||
requires:
|
||||
- linux-checkout
|
||||
context: release-env
|
||||
|
||||
publish-x64-linux:
|
||||
when: << pipeline.parameters.run-linux-x64-publish >>
|
||||
jobs:
|
||||
|
|
@ -2185,6 +2387,17 @@ workflows:
|
|||
- mas-publish:
|
||||
context: release-env
|
||||
|
||||
publish-macos:
|
||||
when: << pipeline.parameters.run-macos-publish >>
|
||||
jobs:
|
||||
- mac-checkout
|
||||
- osx-publish-skip-checkout:
|
||||
requires:
|
||||
- mac-checkout
|
||||
- mas-publish-skip-checkout:
|
||||
requires:
|
||||
- mac-checkout
|
||||
|
||||
lint:
|
||||
when: << pipeline.parameters.run-lint >>
|
||||
jobs:
|
||||
|
|
|
|||
|
|
@ -23,6 +23,11 @@ const circleCIJobs = [
|
|||
'osx-publish'
|
||||
]
|
||||
|
||||
const circleCIPublishWorkflows = [
|
||||
'linux-publish',
|
||||
'macos-publish'
|
||||
]
|
||||
|
||||
const vstsArmJobs = [
|
||||
'electron-arm-testing',
|
||||
'electron-arm64-testing',
|
||||
|
|
@ -87,13 +92,18 @@ async function circleCIcall (targetBranch, job, options) {
|
|||
if (workflowId === -1) {
|
||||
return
|
||||
}
|
||||
console.log(`CircleCI release build workflow running at https://circleci.com/workflow-run/${workflowId} for ${job}.`)
|
||||
const jobNumber = await getCircleCIJobNumber(workflowId)
|
||||
if (jobNumber === -1) {
|
||||
return
|
||||
const workFlowUrl = `https://circleci.com/workflow-run/${workflowId}`
|
||||
if (options.runningPublishWorkflows) {
|
||||
console.log(`CircleCI release workflow request for ${job} successful. Check ${workFlowUrl} for status.`)
|
||||
} else {
|
||||
console.log(`CircleCI release build workflow running at https://circleci.com/workflow-run/${workflowId} for ${job}.`)
|
||||
const jobNumber = await getCircleCIJobNumber(workflowId)
|
||||
if (jobNumber === -1) {
|
||||
return
|
||||
}
|
||||
const jobUrl = `https://circleci.com/gh/electron/electron/${jobNumber}`
|
||||
console.log(`CircleCI release build request for ${job} successful. Check ${jobUrl} for status.`)
|
||||
}
|
||||
const jobUrl = `https://circleci.com/gh/electron/electron/${jobNumber}`
|
||||
console.log(`CircleCI release build request for ${job} successful. Check ${jobUrl} for status.`)
|
||||
} catch (err) {
|
||||
console.log('Error calling CircleCI: ', err)
|
||||
}
|
||||
|
|
@ -232,7 +242,8 @@ function buildCircleCI (targetBranch, options) {
|
|||
assert(circleCIJobs.includes(options.job), `Unknown CircleCI job name: ${options.job}. Valid values are: ${circleCIJobs}.`)
|
||||
circleCIcall(targetBranch, options.job, options)
|
||||
} else {
|
||||
circleCIJobs.forEach((job) => circleCIcall(targetBranch, job, options))
|
||||
options.runningPublishWorkflows = true
|
||||
circleCIPublishWorkflows.forEach((job) => circleCIcall(targetBranch, job, options))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче